Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 0 additions & 35 deletions .github/workflows/ai-review.yml

This file was deleted.

30 changes: 26 additions & 4 deletions .github/workflows/go.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,35 @@ on:

jobs:
unit-tests:
name: ${{ matrix.name }}
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
include:
- name: linux/amd64
os: ubuntu-latest
asset: libduckdb-linux-amd64.zip
- name: linux/arm64
os: ubuntu-24.04-arm
asset: libduckdb-linux-arm64.zip
- name: macos/arm64
os: macos-latest
asset: libduckdb-osx-universal.zip
- name: macos/amd64
os: macos-15-intel
asset: libduckdb-osx-universal.zip
- name: windows/amd64
os: windows-latest
asset: libduckdb-windows-amd64.zip
- name: windows/arm64
os: windows-11-arm
asset: libduckdb-windows-arm64.zip

runs-on: ${{ matrix.os }}

env:
DUCKDB_VERSION: v1.5.4

steps:
- uses: actions/checkout@v4

Expand All @@ -29,23 +51,23 @@ jobs:
- name: Install DuckDB library (Linux)
if: runner.os == 'Linux'
run: |
curl -sSL https://github.com/duckdb/duckdb/releases/download/v1.5.4/libduckdb-linux-amd64.zip -o archive.zip
curl -sSL https://github.com/duckdb/duckdb/releases/download/${DUCKDB_VERSION}/${{ matrix.asset }} -o archive.zip
sudo unzip -j archive.zip libduckdb.so -d /usr/local/lib
sudo ldconfig
rm archive.zip

- name: Install DuckDB library (macOS)
if: runner.os == 'macOS'
run: |
curl -sSL https://github.com/duckdb/duckdb/releases/download/v1.5.4/libduckdb-osx-universal.zip -o archive.zip
curl -sSL https://github.com/duckdb/duckdb/releases/download/${DUCKDB_VERSION}/${{ matrix.asset }} -o archive.zip
sudo unzip -j archive.zip libduckdb.dylib -d /usr/local/lib
rm archive.zip

- name: Install DuckDB library (Windows)
if: runner.os == 'Windows'
shell: pwsh
run: |
Invoke-WebRequest -Uri https://github.com/duckdb/duckdb/releases/download/v1.5.4/libduckdb-windows-amd64.zip -OutFile archive.zip
Invoke-WebRequest -Uri "https://github.com/duckdb/duckdb/releases/download/$env:DUCKDB_VERSION/${{ matrix.asset }}" -OutFile archive.zip
Expand-Archive -Path archive.zip -DestinationPath "$env:RUNNER_TEMP\duckdb" -Force
# go test runs each package with its own directory as the working directory,
# so the DLL must be resolvable via the standard DLL search path (PATH), not cwd.
Expand Down
16 changes: 13 additions & 3 deletions .github/workflows/integ.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,17 @@ on:
- main

jobs:
unit-tests:
integration-tests:
name: ${{ matrix.name }}
strategy:
fail-fast: false
matrix:
include:
- name: glibc
dockerfile: ./internal/integ/Dockerfile
- name: musl
dockerfile: ./internal/integ/Dockerfile.musl

runs-on: ubuntu-latest

steps:
Expand All @@ -19,7 +29,7 @@ jobs:
uses: docker/setup-buildx-action@v3

- name: Build docker image
run: docker build -t go-pduckdb/integ -f ./internal/integ/Dockerfile .
run: docker build -t go-pduckdb/integ-${{ matrix.name }} -f ${{ matrix.dockerfile }} .

- name: Run integration tests
run: docker run --rm go-pduckdb/integ
run: docker run --rm go-pduckdb/integ-${{ matrix.name }}
26 changes: 24 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,4 +1,18 @@
.PHONY: integ
# go-pduckdb — a DuckDB driver for Go that needs no CGO.
#
# These targets are convenience, and they assume GNU make and a POSIX shell:
# Linux, macOS, or WSL / Git Bash on Windows. They are NOT the build.
#
# The build and the tests are the Go toolchain alone and run natively on all
# three, on amd64 and arm64 -- CI proves it on six targets, and it invokes `go`
# directly rather than make for exactly that reason. On Windows:
#
# go test ./...
#
# with duckdb.dll resolvable (see "Library Path Configuration" in the README).
# `make help` and the docker targets are the parts that want a POSIX shell.

.PHONY: run test fmt lint integ integ-arm64 integ-musl integ-musl-arm64 help

run: ## Run the application
CGO_ENABLED=0 go run example/simple/main.go
Expand Down Expand Up @@ -26,6 +40,14 @@ integ-arm64: ## Run integration tests on arm64
docker build --platform linux/arm64 --build-arg GOARCH=arm64 --build-arg LIBARCH=arm64 -t go-pduckdb/integ-arm64 -f internal/integ/Dockerfile . && \
docker run --rm go-pduckdb/integ-arm64

integ-musl: ## Run integration tests against the musl build of DuckDB
docker build --platform linux/amd64 -t go-pduckdb/integ-musl -f internal/integ/Dockerfile.musl . && \
docker run --rm go-pduckdb/integ-musl

integ-musl-arm64: ## Run integration tests against the musl build on arm64
docker build --platform linux/arm64 --build-arg GOARCH=arm64 --build-arg LIBARCH=arm64 -t go-pduckdb/integ-musl-arm64 -f internal/integ/Dockerfile.musl . && \
docker run --rm go-pduckdb/integ-musl-arm64

help: ## Display this help
@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \
@grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \
awk 'BEGIN {FS = ":.*?## "}; {printf "%-20s %s\n", $$1, $$2}'
25 changes: 22 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,16 @@
# go-pduckdb is a PureGO driver for [DuckDB](https://duckdb.org/docs/stable/clients/c/api.html)

> **This is a fork** of [fpt/go-pduckdb](https://github.com/fpt/go-pduckdb) by Youichi
> Fujimoto (MIT), maintained so that Linux, macOS and Windows are all first-class:
> windows/arm64 support, a compile-time check on the Windows struct-by-value ABI
> workaround, macOS amd64 and Linux arm64 back in CI, and a musl build proven
> rather than assumed. See [docs/COMPATIBILITY.md](docs/COMPATIBILITY.md).
>
> The module path is `github.com/calvinchengx/go-pduckdb`; everything else
> matches upstream. Changes are offered back — see
> [fpt/go-pduckdb#37](https://github.com/fpt/go-pduckdb/pull/37) — and this fork
> exists to be merged out of existence if they land.

## Introduction

A DuckDB module for Go which doesn't require CGO.
Expand Down Expand Up @@ -32,7 +43,7 @@ See [docs/COMPATIBILITY.md](./docs/COMPATIBILITY.md) for the full supported feat
## Installation

```bash
go get github.com/fpt/go-pduckdb
go get github.com/calvinchengx/go-pduckdb
```

Also, make sure to install DuckDB on your platform:
Expand Down Expand Up @@ -60,7 +71,15 @@ For other Linux, Check official instruction: [Building DuckDB](https://duckdb.or
### Windows
Download the DuckDB CLI from the [official website](https://duckdb.org/docs/installation/) and place the DLL in your system path.

Note: Windows support relies on an ABI-level workaround for purego's lack of struct-by-value arguments on Windows — see the [Windows workaround](./docs/COMPATIBILITY.md#windows-workaround) section in the compatibility docs.
Note: Windows support relies on an ABI-level workaround for purego's lack of struct-by-value arguments on Windows — see the [Windows workaround](./docs/COMPATIBILITY.md#windows-workaround) section in the compatibility docs. Both amd64 and arm64 are supported and both run the unit tests in CI on every push.

Once `duckdb.dll` is resolvable, the tests are the Go toolchain and nothing else:

```powershell
go test ./...
```

The `Makefile` is convenience rather than the build — its targets assume GNU make and a POSIX shell, so on Windows they want WSL or Git Bash. Nothing is lost by skipping it: `make test` is `go test ./...`, and the remaining targets build Docker images or run `gofumpt` and `golangci-lint`. CI invokes `go` directly for the same reason, which is what makes the Windows results mean anything.

## Library Path Configuration

Expand Down Expand Up @@ -99,7 +118,7 @@ import (
"fmt"
"log"

_ "github.com/fpt/go-pduckdb" // Import for driver registration
_ "github.com/calvinchengx/go-pduckdb" // Import for driver registration
)

func main() {
Expand Down
77 changes: 60 additions & 17 deletions docs/COMPATIBILITY.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,11 @@ Legend: ✅ supported · ⚠️ supported with caveats · ❌ not supported (yet

| Platform | Status | CI-tested | Notes |
|---|---|---|---|
| Linux (amd64/arm64) | ✅ | amd64 | Requires purego ≥ v0.10.0 (struct-by-value support) |
| macOS (amd64/arm64) | ✅ | arm64 | |
| Windows (amd64) | ⚠️ | amd64 | Works via an ABI workaround — see [Windows workaround](#windows-workaround) |
| Linux (amd64/arm64) | ✅ | both | Requires purego ≥ v0.10.0 (struct-by-value support) |
| Linux musl (amd64/arm64) | ✅ | amd64 | DuckDB publishes a musl build; the image must also carry `libstdc++` |
| macOS (amd64/arm64) | ✅ | both | |
| Windows (amd64/arm64) | ✅ | both | Works via an ABI workaround — see [Windows workaround](#windows-workaround) |
| Windows (386/arm) | ❌ | — | Refused at build time; see below |
| FreeBSD / NetBSD | ⚠️ | no | Compiles (covered by build tags), untested |

### Windows workaround
Expand All @@ -21,21 +23,61 @@ API functions take a `duckdb_result` struct by value:
- `duckdb_fetch_chunk`
- `duckdb_result_return_type`

The driver works around this by relying on a Win64 calling-convention detail:
aggregates larger than 8 bytes are passed as a **hidden pointer** to a
caller-allocated copy. Since `duckdb_result` is 48 bytes, at the ABI level these
functions actually receive a `duckdb_result *`. On Windows the driver registers
them with an explicit pointer argument and wraps them to preserve the by-value
signature used on other platforms (see
`internal/duckdb/register_result_windows.go`).
The driver works around this by relying on a property both 64-bit Windows
calling conventions share: an aggregate too large for registers is passed as a
**hidden pointer** to a caller-allocated copy. On amd64 (Win64) that means
anything other than 1, 2, 4 or 8 bytes; on arm64 (AAPCS64) anything over 16
bytes. `duckdb_result` is 48 bytes, so at the ABI level these functions actually
receive a `duckdb_result *`. The driver registers them with an explicit pointer
argument and wraps them to preserve the by-value signature used on other
platforms (see `internal/duckdb/register_result_windows.go`).

This is an ABI-level workaround, not an officially supported purego feature, so
the assumption it rests on is checked rather than trusted:

- The per-architecture threshold is stated explicitly in
`internal/duckdb/abi_windows_amd64.go` and `abi_windows_arm64.go`.
- A **compile-time assertion** fails the build if `duckdb_result` is ever
reduced to a size the convention would pass in registers, because registering
a by-value function with a pointer argument would then read the wrong memory
silently. Note this can only check the Go mirror of the struct: the C API
offers no way to ask for the real size, so a mirror that has drifted from the
C definition is caught by the tests, not by the assertion.
- 32-bit Windows (`386`, `arm`) **fails to build**. Those conventions push large
aggregates onto the stack by value, so the workaround would be wrong rather
than merely unsupported — and DuckDB publishes no 32-bit Windows library.

Both 64-bit Windows targets run the unit tests in CI on every push.

### DLL discovery on Windows

In order: `DUCKDB_LIBRARY_PATH`, `duckdb.dll` beside the **executable**, then in
the current directory, then `%ProgramFiles%\DuckDB\` and
`%ProgramFiles(x86)%\DuckDB\`, and finally the bare name `duckdb.dll` through
the standard `LoadLibrary` search path (`PATH`).

The executable's own directory comes before the working directory because it is
the one location a shipped binary can rely on; the working directory changes
with however the program was launched.

An **absolute** path is loaded with `LoadLibraryExW` restricted to the DLL's own
directory and the process's default directories. That is safer than the legacy
search, which reaches the current directory and `PATH`, and it also lets a
DuckDB DLL resolve dependencies sitting beside it. A bare name still uses the
standard search so that putting a directory on `PATH` keeps working.

## Opening a database

This is an ABI-level workaround, not an officially supported purego feature. It
is exercised in CI on `windows-latest` (amd64) on every push, but be aware of it
if you hit Windows-specific crashes around result fetching.
| Feature | Status | Notes |
|---|---|---|
| Path | ✅ | `sql.Open("duckdb", "warehouse.duckdb")`, or `:memory:` |
| Configuration options | ✅ | As a DSN query string: `"warehouse.duckdb?access_mode=READ_ONLY&threads=2"` |
| Read-only | ✅ | `access_mode=READ_ONLY` — the database refuses writes, rather than the caller intending not to make any |

DLL discovery on Windows: `DUCKDB_LIBRARY_PATH`, then `duckdb.dll` in the
current directory, `%ProgramFiles%\DuckDB\`, `%ProgramFiles(x86)%\DuckDB\`, and
finally the standard `LoadLibrary` search path (`PATH`).
Options go through `duckdb_open_ext`; a path with no `?` uses `duckdb_open`
unchanged. The **last** `?` separates path from options, so a database file
whose name contains one is still reachable. An option DuckDB does not
recognise fails the open with DuckDB's own message rather than being ignored.

## database/sql driver interface

Expand All @@ -51,7 +93,7 @@ finally the standard `LoadLibrary` search path (`PATH`).
| `ColumnTypes()`: database type name | ✅ | `RowsColumnTypeDatabaseTypeName` (e.g. `INTEGER`, `VARCHAR`) |
| `ColumnTypes()`: nullable | ✅ | `RowsColumnTypeNullable` |
| `ColumnTypes()`: precision & scale | ✅ | `RowsColumnTypePrecisionScale` |
| `Result.RowsAffected()` | ⚠️ | Broken for parameterized `Exec` — see [#23](https://github.com/fpt/go-pduckdb/issues/23) |
| `Result.RowsAffected()` | ⚠️ | Broken for parameterized `Exec` — see [#23](https://github.com/calvinchengx/go-pduckdb/issues/23) |
| `Result.LastInsertId()` | ❌ | Not supported by DuckDB; returns an error |
| Named parameters | ❌ | Positional only |

Expand Down Expand Up @@ -102,3 +144,4 @@ Result values are decoded through DuckDB's data-chunk / vector API.
| Go | ≥ 1.24 | |
| purego | ≥ v0.10.0 | v0.10.0 added struct-by-value arguments on Linux, needed for `duckdb_fetch_chunk` |
| DuckDB shared library | v1.5.x | CI tests against v1.5.4; nearby versions generally work since the C API is stable |
| `libstdc++` | — | Only on musl images (Alpine, distroless static): `libduckdb.so` links against it and a musl base does not carry it |
39 changes: 36 additions & 3 deletions driver.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,13 @@ import (
"database/sql"
"database/sql/driver"
"io"
"net/url"
"reflect"
"strings"

"github.com/pkg/errors"

"github.com/fpt/go-pduckdb/internal/duckdb"
"github.com/calvinchengx/go-pduckdb/internal/duckdb"
)

// Initialize and register the driver
Expand All @@ -20,10 +22,41 @@ func init() {
// Driver implements database/sql/driver.Driver
type Driver struct{}

// splitDSN separates the database path from DuckDB configuration options.
//
// The last `?` separates, not the first: a DuckDB path may legitimately
// contain one, and taking the first would make such a file unopenable through
// database/sql with no way to say otherwise.
func splitDSN(dsn string) (string, map[string]string) {
at := strings.LastIndex(dsn, "?")
if at < 0 {
return dsn, nil
}
query, err := url.ParseQuery(dsn[at+1:])
if err != nil || len(query) == 0 {
return dsn, nil
}
settings := make(map[string]string, len(query))
for name, values := range query {
settings[name] = values[len(values)-1]
}
return dsn[:at], settings
}

// Open returns a new connection to the database.
// The dsn is a connection string for the database.
//
// The dsn is the path to the database file, optionally followed by DuckDB
// configuration options as a query string:
//
// sql.Open("duckdb", "warehouse.duckdb?access_mode=READ_ONLY")
// sql.Open("duckdb", ":memory:")
//
// A path containing a `?` that is not meant as options can be written
// `./odd?name.duckdb?` -- the LAST `?` separates. Without one, the whole
// string is the path, so existing callers are unaffected.
func (d *Driver) Open(dsn string) (driver.Conn, error) {
db, err := NewDuckDB(dsn)
path, settings := splitDSN(dsn)
db, err := NewDuckDBWithSettings(path, settings)
if err != nil {
return nil, err
}
Expand Down
2 changes: 1 addition & 1 deletion driver_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import (
"testing"
"time"

"github.com/fpt/go-pduckdb/internal/duckdb"
"github.com/calvinchengx/go-pduckdb/internal/duckdb"
"github.com/stretchr/testify/assert"
)

Expand Down
Loading