From 400fd4693bf72c9da39cb88b4648e4ec6872e2c5 Mon Sep 17 00:00:00 2001 From: JT Archie Date: Tue, 1 Sep 2026 19:31:57 -0600 Subject: [PATCH] add VFSFind so a VFS can delegate temp files to the base VFS A VFS that cannot create files has nowhere to put the temp file sqlite opens when a sorter or a transient index spills, and there is no way to reach the VFS underneath it to ask. The open fails, and with it the query: "unable to open database file". The workaround is to require `PRAGMA temp_store = memory` of every caller, which a library cannot enforce. sqlite's own appendvfs diverts anything that is not the main database to ORIGVFS(pVfs)->xOpen. VFSFind makes the same thing possible here: it wraps sqlite3_vfs_find and returns the registered VFS as a VFS this package can call, so an implementation can pass on the files it cannot serve. The C side wraps the sqlite3_vfs and sqlite3_io_methods function pointers; the Go side is a VFS/File pair over them. The sqlite3_file is allocated with sqlite3_malloc, since sqlite holds it for the life of the file and Go memory may not be used for that. ReadAt reports a short read as (len(p), io.EOF). xRead does not say how many bytes it read, only that it zero-filled the rest, and the bytes it did read are in the buffer -- returning a smaller n would get them overwritten with zeros on the way back to sqlite. Fixes #15 Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01WMBg7JBvZJRRu1RytUj5c3 --- basevfs.go | 229 ++++++++++++++++++++++++++++++++++++++++++++++++ basevfs_test.go | 123 ++++++++++++++++++++++++++ sqlite3vfs.c | 93 ++++++++++++++++++++ sqlite3vfs.h | 25 ++++++ 4 files changed, 470 insertions(+) create mode 100644 basevfs.go create mode 100644 basevfs_test.go diff --git a/basevfs.go b/basevfs.go new file mode 100644 index 0000000..719eadd --- /dev/null +++ b/basevfs.go @@ -0,0 +1,229 @@ +package sqlite3vfs + +/* + #include "sqlite3vfs.h" + #include +*/ +import "C" + +import ( + "io" + "unsafe" +) + +// VFSFind returns the VFS sqlite has registered under name, wrapped so that a +// Go VFS in this package can delegate to it. An empty name returns sqlite's +// current default VFS. It returns nil when no VFS is registered under that +// name. +// +// This exists for VFS implementations that cannot serve every file sqlite asks +// them for. A read-only or compressed VFS, for example, has nowhere to put the +// temp file sqlite opens when a sorter or a transient index spills, and failing +// that open fails the query. Such a VFS can pass anything that is not the main +// database down to the VFS underneath it, which is what sqlite's own appendvfs +// does: +// +// func (vfs *MyVFS) Open(name string, flags sqlite3vfs.OpenFlag) (sqlite3vfs.File, sqlite3vfs.OpenFlag, error) { +// if flags&sqlite3vfs.OpenMainDB == 0 { +// return vfs.base.Open(name, flags) +// } +// // ... open the main database yourself +// } +// +// The returned VFS is a handle onto a VFS sqlite already owns; there is nothing +// to close, and it must not be passed to RegisterVFS. +func VFSFind(name string) VFS { + var cName *C.char + + if name != "" { + cName = C.CString(name) + defer C.free(unsafe.Pointer(cName)) + } + + vfs := C.s3vfsFind(cName) + if vfs == nil { + return nil + } + + return &baseVFS{vfs: vfs} +} + +// baseVFS is a VFS backed by a sqlite3_vfs registered in sqlite itself. +type baseVFS struct { + vfs *C.sqlite3_vfs +} + +var _ VFS = (*baseVFS)(nil) + +// MaxPathname reports the longest path the underlying VFS accepts. +func (v *baseVFS) MaxPathname() int { + return int(C.s3vfsBaseMaxPathname(v.vfs)) +} + +func (v *baseVFS) Open(name string, flags OpenFlag) (File, OpenFlag, error) { + // A nil name (not a pointer to "") is how sqlite asks for a temp file. + var cName *C.char + + if name != "" { + cName = C.CString(name) + defer C.free(unsafe.Pointer(cName)) + } + + file := C.s3vfsBaseFileAlloc(v.vfs) + if file == nil { + return nil, 0, NoMemError + } + + var outFlags C.int + + rc := C.s3vfsBaseOpen(v.vfs, cName, file, C.int(flags), &outFlags) + if int(rc) != sqliteOK { + C.s3vfsBaseFileFree(file) + + return nil, 0, errFromCode(int(rc)) + } + + return &baseFile{file: file}, OpenFlag(outFlags), nil +} + +func (v *baseVFS) Delete(name string, dirSync bool) error { + cName := C.CString(name) + defer C.free(unsafe.Pointer(cName)) + + var syncDir C.int + if dirSync { + syncDir = 1 + } + + return errFromCode(int(C.s3vfsBaseDelete(v.vfs, cName, syncDir))) +} + +func (v *baseVFS) Access(name string, flags AccessFlag) (bool, error) { + cName := C.CString(name) + defer C.free(unsafe.Pointer(cName)) + + var out C.int + + rc := C.s3vfsBaseAccess(v.vfs, cName, C.int(flags), &out) + if int(rc) != sqliteOK { + return false, errFromCode(int(rc)) + } + + return out != 0, nil +} + +// FullPathname returns the name unchanged on error, since this interface has +// no way to report one. +func (v *baseVFS) FullPathname(name string) string { + cName := C.CString(name) + defer C.free(unsafe.Pointer(cName)) + + size := int(C.s3vfsBaseMaxPathname(v.vfs)) + 1 + out := (*C.char)(C.malloc(C.size_t(size))) + defer C.free(unsafe.Pointer(out)) + + rc := C.s3vfsBaseFullPathname(v.vfs, cName, C.int(size), out) + if int(rc) != sqliteOK { + return name + } + + return C.GoString(out) +} + +// baseFile is a File backed by a sqlite3_file belonging to another VFS. +type baseFile struct { + file *C.sqlite3_file +} + +var _ File = (*baseFile)(nil) + +func (f *baseFile) Close() error { + if f.file == nil { + return nil + } + + rc := C.s3vfsBaseClose(f.file) + C.s3vfsBaseFileFree(f.file) + f.file = nil + + return errFromCode(int(rc)) +} + +// ReadAt reports a short read as (len(p), io.EOF): xRead zero-fills the tail +// and keeps the bytes it read in p, so a smaller n would get them overwritten +// with zeros on the way back to sqlite. +func (f *baseFile) ReadAt(p []byte, off int64) (int, error) { + if len(p) == 0 { + return 0, nil + } + + rc := int(C.s3vfsBaseRead(f.file, unsafe.Pointer(&p[0]), C.int(len(p)), C.sqlite3_int64(off))) + + switch rc { + case sqliteOK: + return len(p), nil + case IOErrorShortRead.code: + return len(p), io.EOF + default: + return 0, errFromCode(rc) + } +} + +func (f *baseFile) WriteAt(p []byte, off int64) (int, error) { + if len(p) == 0 { + return 0, nil + } + + rc := int(C.s3vfsBaseWrite(f.file, unsafe.Pointer(&p[0]), C.int(len(p)), C.sqlite3_int64(off))) + if rc != sqliteOK { + return 0, errFromCode(rc) + } + + return len(p), nil +} + +func (f *baseFile) Truncate(size int64) error { + return errFromCode(int(C.s3vfsBaseTruncate(f.file, C.sqlite3_int64(size)))) +} + +func (f *baseFile) Sync(flag SyncType) error { + return errFromCode(int(C.s3vfsBaseSync(f.file, C.int(flag)))) +} + +func (f *baseFile) FileSize() (int64, error) { + var size C.sqlite3_int64 + + rc := C.s3vfsBaseFileSize(f.file, &size) + if int(rc) != sqliteOK { + return 0, errFromCode(int(rc)) + } + + return int64(size), nil +} + +func (f *baseFile) Lock(elock LockType) error { + return errFromCode(int(C.s3vfsBaseLock(f.file, C.int(elock)))) +} + +func (f *baseFile) Unlock(elock LockType) error { + return errFromCode(int(C.s3vfsBaseUnlock(f.file, C.int(elock)))) +} + +func (f *baseFile) CheckReservedLock() (bool, error) { + var out C.int + + rc := C.s3vfsBaseCheckReservedLock(f.file, &out) + if int(rc) != sqliteOK { + return false, errFromCode(int(rc)) + } + + return out != 0, nil +} + +func (f *baseFile) SectorSize() int64 { + return int64(C.s3vfsBaseSectorSize(f.file)) +} + +func (f *baseFile) DeviceCharacteristics() DeviceCharacteristic { + return DeviceCharacteristic(C.s3vfsBaseDeviceCharacteristics(f.file)) +} diff --git a/basevfs_test.go b/basevfs_test.go new file mode 100644 index 0000000..24b7118 --- /dev/null +++ b/basevfs_test.go @@ -0,0 +1,123 @@ +package sqlite3vfs + +import ( + "database/sql" + "fmt" + "testing" + + _ "github.com/mattn/go-sqlite3" +) + +func TestVFSFindDefault(t *testing.T) { + if VFSFind("") == nil { + t.Fatal("VFSFind(\"\") returned nil; sqlite always has a default VFS") + } + + if vfs := VFSFind("no-such-vfs"); vfs != nil { + t.Fatalf("VFSFind of an unregistered name returned %v, want nil", vfs) + } +} + +// noTempVFS cannot make a temp file of its own and hands that open to base; +// with a nil base the open fails, as it did before VFSFind existed. +type noTempVFS struct { + *TmpVFS + base VFS +} + +func (vfs *noTempVFS) Open(name string, flags OpenFlag) (File, OpenFlag, error) { + if name == "" { + if vfs.base == nil { + return nil, 0, CantOpenError + } + + return vfs.base.Open(name, flags) + } + + return vfs.TmpVFS.Open(name, flags) +} + +// TestDelegateTempFiles runs a query that must spill to a temp file against a +// VFS that cannot make one: it fails without a base VFS to delegate to and +// succeeds with one. +func TestDelegateTempFiles(t *testing.T) { + for _, tc := range []struct { + name string + base VFS + wantError bool + }{ + {name: "no_delegate", base: nil, wantError: true}, + {name: "delegate_to_default", base: VFSFind("")}, + } { + t.Run(tc.name, func(t *testing.T) { + vfsName := "notemp_" + tc.name + if err := RegisterVFS(vfsName, &noTempVFS{TmpVFS: newTempVFS(), base: tc.base}); err != nil { + t.Fatal(err) + } + + db, err := sql.Open("sqlite3", fmt.Sprintf("spill.db?vfs=%s", vfsName)) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + // One connection so the PRAGMAs apply where the query runs; + // temp_store=FILE plus a tiny cache forces the sort to spill. + db.SetMaxOpenConns(1) + + for _, pragma := range []string{"PRAGMA temp_store = FILE", "PRAGMA cache_size = -16"} { + if _, err := db.Exec(pragma); err != nil { + t.Fatalf("%s: %s", pragma, err) + } + } + + if _, err := db.Exec(`CREATE TABLE spill (id INTEGER PRIMARY KEY, v TEXT)`); err != nil { + t.Fatal(err) + } + + tx, err := db.Begin() + if err != nil { + t.Fatal(err) + } + + stmt, err := tx.Prepare(`INSERT INTO spill (v) VALUES (?)`) + if err != nil { + t.Fatal(err) + } + + for i := 0; i < 20000; i++ { + if _, err := stmt.Exec(fmt.Sprintf("row-%d-padded-padded-padded-padded", i)); err != nil { + t.Fatal(err) + } + } + + stmt.Close() + + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + + rows, err := db.Query(`SELECT v, COUNT(*) FROM spill GROUP BY v ORDER BY 2, 1`) + if err == nil { + for rows.Next() { + } + err = rows.Err() + rows.Close() + } + + if tc.wantError { + if err == nil { + t.Fatal("spilling query succeeded with no VFS to open a temp file on") + } + + t.Logf("without delegation: %s", err) + + return + } + + if err != nil { + t.Fatalf("spilling query failed with delegation in place: %s", err) + } + }) + } +} diff --git a/sqlite3vfs.c b/sqlite3vfs.c index 6abd1eb..ce1cb46 100644 --- a/sqlite3vfs.c +++ b/sqlite3vfs.c @@ -1,6 +1,7 @@ #include "sqlite3vfs.h" #include #include +#include #ifdef SQLITE3VFS_LOADABLE_EXT SQLITE_EXTENSION_INIT1 @@ -174,3 +175,95 @@ const sqlite3_io_methods s3vfs_io_methods = { s3vfsSectorSize, /* xSectorSize */ s3vfsDeviceCharacteristics, /* xDeviceCharacteristics */ }; + +/* Wrappers making a registered sqlite3_vfs's function pointers callable from +** Go, so a Go VFS can delegate the files it cannot serve. */ + +sqlite3_vfs* s3vfsFind(const char* zName) { + return sqlite3_vfs_find(zName); +} + +int s3vfsBaseOsFileSize(sqlite3_vfs* vfs) { + return vfs->szOsFile; +} + +int s3vfsBaseMaxPathname(sqlite3_vfs* vfs) { + return vfs->mxPathname; +} + +/* sqlite3_malloc, not Go memory: sqlite holds this pointer for the life of +** the file. */ +sqlite3_file* s3vfsBaseFileAlloc(sqlite3_vfs* vfs) { + sqlite3_file* file = (sqlite3_file*)sqlite3_malloc(vfs->szOsFile); + if (file != NULL) { + memset(file, 0, vfs->szOsFile); + } + return file; +} + +void s3vfsBaseFileFree(sqlite3_file* file) { + sqlite3_free(file); +} + +int s3vfsBaseOpen(sqlite3_vfs* vfs, const char* zName, sqlite3_file* file, int flags, int* outFlags) { + return vfs->xOpen(vfs, zName, file, flags, outFlags); +} + +int s3vfsBaseDelete(sqlite3_vfs* vfs, const char* zName, int syncDir) { + return vfs->xDelete(vfs, zName, syncDir); +} + +int s3vfsBaseAccess(sqlite3_vfs* vfs, const char* zName, int flags, int* pResOut) { + return vfs->xAccess(vfs, zName, flags, pResOut); +} + +int s3vfsBaseFullPathname(sqlite3_vfs* vfs, const char* zName, int nOut, char* zOut) { + return vfs->xFullPathname(vfs, zName, nOut, zOut); +} + +int s3vfsBaseClose(sqlite3_file* file) { + if (file->pMethods == NULL) { + return SQLITE_OK; + } + return file->pMethods->xClose(file); +} + +int s3vfsBaseRead(sqlite3_file* file, void* buf, int iAmt, sqlite3_int64 iOfst) { + return file->pMethods->xRead(file, buf, iAmt, iOfst); +} + +int s3vfsBaseWrite(sqlite3_file* file, const void* buf, int iAmt, sqlite3_int64 iOfst) { + return file->pMethods->xWrite(file, buf, iAmt, iOfst); +} + +int s3vfsBaseTruncate(sqlite3_file* file, sqlite3_int64 size) { + return file->pMethods->xTruncate(file, size); +} + +int s3vfsBaseSync(sqlite3_file* file, int flags) { + return file->pMethods->xSync(file, flags); +} + +int s3vfsBaseFileSize(sqlite3_file* file, sqlite3_int64* pSize) { + return file->pMethods->xFileSize(file, pSize); +} + +int s3vfsBaseLock(sqlite3_file* file, int eLock) { + return file->pMethods->xLock(file, eLock); +} + +int s3vfsBaseUnlock(sqlite3_file* file, int eLock) { + return file->pMethods->xUnlock(file, eLock); +} + +int s3vfsBaseCheckReservedLock(sqlite3_file* file, int* pResOut) { + return file->pMethods->xCheckReservedLock(file, pResOut); +} + +int s3vfsBaseSectorSize(sqlite3_file* file) { + return file->pMethods->xSectorSize(file); +} + +int s3vfsBaseDeviceCharacteristics(sqlite3_file* file) { + return file->pMethods->xDeviceCharacteristics(file); +} diff --git a/sqlite3vfs.h b/sqlite3vfs.h index 2e4747b..c10637f 100644 --- a/sqlite3vfs.h +++ b/sqlite3vfs.h @@ -52,6 +52,31 @@ const extern sqlite3_io_methods s3vfs_io_methods; char* s3vfsSqlite3Mprintf(const char* str); +/* Wrappers over a registered sqlite3_vfs's function pointers, so a Go VFS can +** delegate work (temp files, most commonly) to the VFS underneath it. */ +sqlite3_vfs* s3vfsFind(const char* zName); +int s3vfsBaseOsFileSize(sqlite3_vfs* vfs); +int s3vfsBaseMaxPathname(sqlite3_vfs* vfs); +sqlite3_file* s3vfsBaseFileAlloc(sqlite3_vfs* vfs); +void s3vfsBaseFileFree(sqlite3_file* file); + +int s3vfsBaseOpen(sqlite3_vfs* vfs, const char* zName, sqlite3_file* file, int flags, int* outFlags); +int s3vfsBaseDelete(sqlite3_vfs* vfs, const char* zName, int syncDir); +int s3vfsBaseAccess(sqlite3_vfs* vfs, const char* zName, int flags, int* pResOut); +int s3vfsBaseFullPathname(sqlite3_vfs* vfs, const char* zName, int nOut, char* zOut); + +int s3vfsBaseClose(sqlite3_file* file); +int s3vfsBaseRead(sqlite3_file* file, void* buf, int iAmt, sqlite3_int64 iOfst); +int s3vfsBaseWrite(sqlite3_file* file, const void* buf, int iAmt, sqlite3_int64 iOfst); +int s3vfsBaseTruncate(sqlite3_file* file, sqlite3_int64 size); +int s3vfsBaseSync(sqlite3_file* file, int flags); +int s3vfsBaseFileSize(sqlite3_file* file, sqlite3_int64* pSize); +int s3vfsBaseLock(sqlite3_file* file, int eLock); +int s3vfsBaseUnlock(sqlite3_file* file, int eLock); +int s3vfsBaseCheckReservedLock(sqlite3_file* file, int* pResOut); +int s3vfsBaseSectorSize(sqlite3_file* file); +int s3vfsBaseDeviceCharacteristics(sqlite3_file* file); + const char* s3vfsURIKey(const char* zName, int n); const char* s3vfsURIParameter(const char* zName, const char* zParam);