From 42a396d43a4453578f71f4d5d40f4b969551047d Mon Sep 17 00:00:00 2001 From: dmgcodevil Date: Sun, 1 Mar 2026 16:32:35 -0500 Subject: [PATCH] linux release --- docker/Dockerfile.linux-release | 191 +++++++++++++++++++++++++ docs/release-linux.md | 243 ++++++++++++++++++++++++++++++++ scripts/create_linux_bundle.sh | 108 ++++++++++++++ 3 files changed, 542 insertions(+) create mode 100644 docker/Dockerfile.linux-release create mode 100644 docs/release-linux.md create mode 100755 scripts/create_linux_bundle.sh diff --git a/docker/Dockerfile.linux-release b/docker/Dockerfile.linux-release new file mode 100644 index 0000000..de11b76 --- /dev/null +++ b/docker/Dockerfile.linux-release @@ -0,0 +1,191 @@ +# --------------------------------------------------------------------------- +# TundraDB — Linux Release Builder +# +# Multi-stage Dockerfile that builds tundra_shell in Release mode and +# produces a self-contained distributable tarball. +# +# Usage (from project root): +# docker build --platform linux/amd64 \ +# -f docker/Dockerfile.linux-release \ +# -t tundradb-linux-release . +# +# docker run --rm -v "$(pwd)/dist:/output" tundradb-linux-release +# +# The tarball will appear in dist/ +# --------------------------------------------------------------------------- + +ARG TARGETPLATFORM=linux/amd64 +FROM --platform=${TARGETPLATFORM} ubuntu:24.04 AS builder + +ARG TUNDRADB_VERSION=1.0.0 + +ENV DEBIAN_FRONTEND=noninteractive +ENV TUNDRADB_VERSION=${TUNDRADB_VERSION} + +# --- 1. Install system packages --- +RUN apt-get update && apt-get install -y \ + build-essential \ + gcc-13 \ + g++-13 \ + git \ + pkg-config \ + python3 \ + wget \ + curl \ + lsb-release \ + gnupg \ + ca-certificates \ + uuid-dev \ + libboost-all-dev \ + libtbb-dev \ + libgtest-dev \ + libcds-dev \ + openjdk-11-jdk \ + patchelf \ + llvm clang lldb \ + file \ + && rm -rf /var/lib/apt/lists/* + +# Set GCC 13 as default +RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 100 \ + && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-13 100 + +# --- 2. Install CMake 3.30 --- +RUN wget -q https://github.com/Kitware/CMake/releases/download/v3.30.0/cmake-3.30.0-linux-x86_64.sh \ + -O /tmp/cmake-install.sh \ + && chmod u+x /tmp/cmake-install.sh \ + && mkdir -p /opt/cmake \ + && /tmp/cmake-install.sh --skip-license --prefix=/opt/cmake \ + && ln -sf /opt/cmake/bin/* /usr/local/bin/ \ + && rm /tmp/cmake-install.sh + +# --- 3. Install Apache Arrow + Parquet --- +RUN wget https://packages.apache.org/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb \ + && apt-get install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb \ + && apt-get update \ + && apt-get install -y -V libarrow-dev libarrow-dataset-dev libparquet-dev \ + && (apt-get install -y libarrow-compute-dev 2>/dev/null || true) \ + && rm -f ./apache-arrow-apt-source-latest-*.deb \ + && rm -rf /var/lib/apt/lists/* + +# --- 4. Install spdlog from source --- +RUN cd /tmp \ + && git clone --depth 1 --branch v1.12.0 https://github.com/gabime/spdlog.git \ + && cd spdlog && mkdir build && cd build \ + && cmake .. -DCMAKE_BUILD_TYPE=Release -DSPDLOG_BUILD_SHARED=ON \ + && make -j$(nproc) && make install \ + && ldconfig && cd / && rm -rf /tmp/spdlog + +# --- 5. Install fmt from source --- +RUN cd /tmp \ + && git clone --depth 1 --branch 10.2.1 https://github.com/fmtlib/fmt.git \ + && cd fmt && mkdir build && cd build \ + && cmake .. -DCMAKE_BUILD_TYPE=Release -DFMT_DOC=OFF -DFMT_TEST=OFF \ + && make -j$(nproc) && make install \ + && ldconfig && cd / && rm -rf /tmp/fmt + +# --- 6. Install Google Benchmark from source --- +RUN cd /tmp \ + && git clone --depth 1 https://github.com/google/benchmark.git \ + && cd benchmark && mkdir build && cd build \ + && cmake .. -DCMAKE_BUILD_TYPE=Release -DBENCHMARK_ENABLE_GTEST_TESTS=OFF \ + && make -j$(nproc) && make install \ + && ldconfig && cd / && rm -rf /tmp/benchmark + +# --- 7. Install ANTLR4 C++ runtime from source --- +RUN cd /tmp \ + && git clone https://github.com/antlr/antlr4.git \ + && cd antlr4/runtime/Cpp && mkdir build && cd build \ + && cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_CXX_STANDARD=17 \ + && make -j$(nproc) && make install \ + && ldconfig \ + && cp -r /usr/local/include/antlr4-runtime/* /usr/local/include/ 2>/dev/null || \ + cp -r /tmp/antlr4/runtime/Cpp/runtime/src/* /usr/local/include/ \ + && cd / && rm -rf /tmp/antlr4 + +# --- 8. Copy source and build --- +WORKDIR /workspace +COPY . . + +RUN cmake -B build_release -S . \ + -DCMAKE_BUILD_TYPE=Release \ + -DTUNDRADB_BUILD_TESTS=OFF \ + -DTUNDRADB_BUILD_BENCHMARKS=OFF \ + -DTUNDRADB_BUILD_SHELL=ON \ + && cmake --build build_release -j$(nproc) + +# --- 9. Bundle into self-contained distribution --- +RUN set -e && \ + APP_NAME="tundra_shell" && \ + BUNDLE="TundraDB-${TUNDRADB_VERSION}-Linux-x86_64" && \ + BUNDLE_DIR="/dist/${BUNDLE}" && \ + \ + mkdir -p "${BUNDLE_DIR}/bin" "${BUNDLE_DIR}/libs" && \ + cp "build_release/${APP_NAME}" "${BUNDLE_DIR}/bin/" && \ + \ + echo "=== Collecting shared libraries ===" && \ + # Collect all .so dependencies, skip system/kernel libs + ldd "${BUNDLE_DIR}/bin/${APP_NAME}" | \ + awk '/=>/ && !/linux-vdso|ld-linux|libc\.so|libm\.so|libpthread|libdl\.so|librt\.so|libresolv|libnss|libnsl/ {print $3}' | \ + sort -u | while read -r lib; do \ + if [ -f "$lib" ]; then \ + cp -L "$lib" "${BUNDLE_DIR}/libs/" ; \ + echo " bundled: $(basename "$lib")" ; \ + fi ; \ + done && \ + \ + # Also check for transitive deps of bundled libs + echo "=== Collecting transitive dependencies ===" && \ + for attempt in 1 2 3; do \ + FOUND_NEW=false ; \ + for bundled_lib in "${BUNDLE_DIR}/libs/"*.so*; do \ + ldd "$bundled_lib" 2>/dev/null | \ + awk '/=>/ && !/linux-vdso|ld-linux|libc\.so|libm\.so|libpthread|libdl\.so|librt\.so|libresolv|libnss|libnsl/ {print $3}' | \ + sort -u | while read -r lib; do \ + base=$(basename "$lib") ; \ + if [ -f "$lib" ] && [ ! -f "${BUNDLE_DIR}/libs/${base}" ]; then \ + cp -L "$lib" "${BUNDLE_DIR}/libs/" ; \ + echo " bundled (transitive pass $attempt): ${base}" ; \ + fi ; \ + done ; \ + done ; \ + done && \ + \ + echo "=== Setting RPATH ===" && \ + # Set RPATH on the binary so it finds libs at ../libs/ relative to itself + patchelf --set-rpath '$ORIGIN/../libs' "${BUNDLE_DIR}/bin/${APP_NAME}" && \ + \ + # Set RPATH on all bundled libs so they find each other + for lib in "${BUNDLE_DIR}/libs/"*.so*; do \ + patchelf --set-rpath '$ORIGIN' "$lib" 2>/dev/null || true ; \ + done && \ + \ + echo "=== Stripping debug symbols ===" && \ + strip --strip-unneeded "${BUNDLE_DIR}/bin/${APP_NAME}" 2>/dev/null || true && \ + for lib in "${BUNDLE_DIR}/libs/"*.so*; do \ + strip --strip-unneeded "$lib" 2>/dev/null || true ; \ + done && \ + \ + # Create launcher script + printf '#!/bin/bash\nSCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"\nexec "${SCRIPT_DIR}/bin/tundra_shell" "$@"\n' \ + > "${BUNDLE_DIR}/${APP_NAME}" && \ + chmod +x "${BUNDLE_DIR}/${APP_NAME}" && \ + \ + # Version file + echo "${TUNDRADB_VERSION}" > "${BUNDLE_DIR}/VERSION" && \ + \ + echo "=== Verifying ===" && \ + ldd "${BUNDLE_DIR}/bin/${APP_NAME}" && \ + \ + echo "=== Creating archive ===" && \ + cd /dist && \ + tar -czf "${BUNDLE}.tar.gz" "${BUNDLE}" && \ + echo "" && \ + echo "=== Bundle complete ===" && \ + echo "Libraries bundled: $(ls -1 "${BUNDLE_DIR}/libs/" | wc -l)" && \ + echo "Archive size: $(du -sh "${BUNDLE}.tar.gz" | awk '{print $1}')" && \ + echo "Archive: /dist/${BUNDLE}.tar.gz" + +# --- 10. Default: copy tarball to /output volume --- +CMD ["sh", "-c", "cp /dist/TundraDB-*.tar.gz /output/ && echo 'Copied to /output/' && ls -lh /output/TundraDB-*.tar.gz"] + diff --git a/docs/release-linux.md b/docs/release-linux.md new file mode 100644 index 0000000..b90af6d --- /dev/null +++ b/docs/release-linux.md @@ -0,0 +1,243 @@ +# TundraDB — Linux Release Process + +## Overview + +This document describes how to build a self-contained Linux (x86_64) +distribution of TundraDB from **any** machine with Docker — including macOS. + +The process uses Docker to spin up an Ubuntu 24.04 build environment, compile +`tundra_shell` in Release mode, bundle all shared library dependencies, and +produce a `.tar.gz` archive that runs on any modern Linux x86_64 system without +needing to install anything. + +### Output structure + +``` +TundraDB-1.0.0-Linux-x86_64.tar.gz +└── TundraDB-1.0.0-Linux-x86_64/ + ├── tundra_shell ← launcher script (run this) + ├── bin/ + │ └── tundra_shell ← ELF binary (RPATH set to $ORIGIN/../libs) + ├── libs/ + │ └── *.so* ← all bundled shared libraries + └── VERSION +``` + +--- + +## Prerequisites + +On the **build machine** (macOS or Linux): + +```bash +# Docker Desktop (macOS) or Docker Engine (Linux) +# https://docs.docker.com/get-docker/ +docker --version # verify it's installed and running +``` + +That's it — all C++ dependencies are installed **inside** the Docker image. + +--- + +## Quick Release (one command) + +```bash +./scripts/create_linux_bundle.sh +``` + +This will: + +1. Build a Docker image (`tundradb-linux-release`) with all build dependencies +2. Compile `tundra_shell` in Release mode inside the container +3. Use `ldd` to collect all non-system `.so` dependencies +4. Use `patchelf` to set `RPATH` to `$ORIGIN/../libs` +5. Strip debug symbols for smaller size +6. Create the tarball inside the image +7. Extract it to `dist/TundraDB-1.0.0-Linux-x86_64.tar.gz` + +### Options + +| Flag | Effect | +|-----------------|---------------------------------------------------------| +| `--no-cache` | Force full Docker rebuild (ignore layer cache) | +| `--skip-build` | Skip Docker build, reuse existing image | + +### Timing + +| Scenario | Approximate time | +|-----------------------------------|--------------------| +| First build (no Docker cache) | 10–20 minutes | +| Rebuild (only source changed) | 1–3 minutes | +| `--skip-build` (extract only) | ~5 seconds | + +> Docker caches each layer. Only the `COPY . .` and subsequent layers are +> rebuilt when source code changes. Dependency installation is cached. + +--- + +## How it works + +### Docker image layers + +``` +┌─────────────────────────────────┐ +│ Ubuntu 24.04 + GCC 13 │ ← cached after first build +│ CMake 3.30 │ +│ Arrow, Parquet, TBB, CDS │ +│ spdlog, fmt, ANTLR4, LLVM │ +│ patchelf │ +├─────────────────────────────────┤ +│ COPY . . (source code) │ ← rebuilt when source changes +│ cmake -B build_release ... │ +│ cmake --build build_release │ +├─────────────────────────────────┤ +│ Bundle phase: │ +│ ldd → collect .so files │ +│ patchelf → set RPATH │ +│ strip → remove debug symbols │ +│ tar → create archive │ +└─────────────────────────────────┘ +``` + +### Library bundling + +On Linux, shared libraries are located via `RPATH` (embedded in the ELF binary) +or `LD_LIBRARY_PATH` (environment variable). We use `patchelf` to set: + +- **Binary**: `RPATH = $ORIGIN/../libs` +- **Bundled libs**: `RPATH = $ORIGIN` + +This makes the binary fully self-contained — it finds its libraries relative to +its own location, with no need to install them system-wide. + +### What's bundled vs. not bundled + +| Bundled (copied to libs/) | NOT bundled (expected on system) | +|------------------------------------|----------------------------------| +| libarrow, libparquet | libc, libm (glibc) | +| libtbb | libpthread, libdl, librt | +| libspdlog, libfmt | ld-linux-x86-64.so (loader) | +| libantlr4-runtime | linux-vdso.so (kernel) | +| libcds | | +| libLLVM*, libstdc++ | | +| all transitive .so dependencies | | + +> The only system requirement is a glibc-based Linux (Ubuntu 20.04+, Debian 11+, +> RHEL 8+, Fedora 33+, etc.). Musl-based distros (Alpine) are not supported. + +--- + +## Step-by-step (manual) + +If you prefer running Docker commands directly: + +### 1. Build the image + +```bash +cd /path/to/tundradb + +docker build --platform linux/amd64 \ + -f docker/Dockerfile.linux-release \ + -t tundradb-linux-release \ + --build-arg TUNDRADB_VERSION=1.0.0 \ + . +``` + +### 2. Extract the tarball + +```bash +mkdir -p dist + +docker create --platform linux/amd64 \ + --name tundradb-extract \ + tundradb-linux-release /bin/true + +docker cp tundradb-extract:/dist/TundraDB-1.0.0-Linux-x86_64.tar.gz dist/ +docker rm tundradb-extract +``` + +### 3. (Optional) Inspect the bundle + +```bash +# List contents +tar tzf dist/TundraDB-1.0.0-Linux-x86_64.tar.gz + +# Inspect RPATH +docker run --rm --platform linux/amd64 tundradb-linux-release \ + patchelf --print-rpath /dist/TundraDB-1.0.0-Linux-x86_64/bin/tundra_shell + +# List linked libraries +docker run --rm --platform linux/amd64 tundradb-linux-release \ + ldd /dist/TundraDB-1.0.0-Linux-x86_64/bin/tundra_shell +``` + +--- + +## Using the package (for end-users) + +On the target Linux machine: + +```bash +# Download / copy the archive +tar xzf TundraDB-1.0.0-Linux-x86_64.tar.gz +cd TundraDB-1.0.0-Linux-x86_64 + +# Run TundraDB shell +./tundra_shell --db-path ./my-database +``` + +No installation, no root, no package manager needed. + +--- + +## Troubleshooting + +### Docker build fails at Arrow installation + +The Arrow apt repository URL may change. Check the +[Apache Arrow docs](https://arrow.apache.org/install/) for the latest +Ubuntu/Debian installation instructions. + +### `qemu: uncaught target signal 11` during build + +This can happen when building `linux/amd64` on Apple Silicon via QEMU emulation. +Solutions: + +1. Increase Docker Desktop memory (Settings → Resources → 8 GB+) +2. Use `--no-cache` to avoid stale layers +3. If persistent, use a remote Linux builder (`docker buildx create --driver remote`) + +### Binary reports "GLIBC not found" on target + +The bundle is built against Ubuntu 24.04 (glibc 2.39). If the target system has +an older glibc, you'll see errors like: + +``` +/lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.38' not found +``` + +**Fix**: Build with an older base image. Change `ubuntu:24.04` to `ubuntu:22.04` +in `docker/Dockerfile.linux-release` (and adjust package names if needed). + +### Archive is too large + +```bash +# Check what's taking space +tar tzf dist/TundraDB-1.0.0-Linux-x86_64.tar.gz | head -30 + +# The script already strips symbols, but you can also compress harder: +cd dist +tar xzf TundraDB-1.0.0-Linux-x86_64.tar.gz +tar -cJf TundraDB-1.0.0-Linux-x86_64.tar.xz TundraDB-1.0.0-Linux-x86_64 +# .tar.xz is ~30% smaller than .tar.gz +``` + +--- + +## Platform Notes + +- This produces an **x86_64** binary. It will run on Intel/AMD 64-bit Linux. + ARM64 Linux support would require building with `--platform linux/arm64`. +- For macOS distribution, see `docs/release-macos.md`. +- The Docker image is ~3–4 GB. The final tarball is typically 20–50 MB. + diff --git a/scripts/create_linux_bundle.sh b/scripts/create_linux_bundle.sh new file mode 100755 index 0000000..29e4875 --- /dev/null +++ b/scripts/create_linux_bundle.sh @@ -0,0 +1,108 @@ +#!/bin/bash +set -euo pipefail + +# --------------------------------------------------------------------------- +# TundraDB Linux Bundle Creator (Docker-based) +# +# Builds a self-contained Linux x86_64 distributable from macOS (or Linux) +# using Docker. Produces a .tar.gz archive with tundra_shell and all its +# shared library dependencies. +# +# Prerequisites: +# - Docker Desktop running (with linux/amd64 support) +# +# Usage: +# ./scripts/create_linux_bundle.sh # build image + extract +# ./scripts/create_linux_bundle.sh --no-cache # rebuild from scratch +# ./scripts/create_linux_bundle.sh --skip-build # reuse existing image +# --------------------------------------------------------------------------- + +VERSION="1.0.0" +IMAGE_NAME="tundradb-linux-release" +CONTAINER_NAME="tundradb-linux-extract" + +NO_CACHE="" +SKIP_BUILD=false + +for arg in "$@"; do + case "$arg" in + --no-cache) NO_CACHE="--no-cache" ;; + --skip-build) SKIP_BUILD=true ;; + esac +done + +# Paths +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +DIST_DIR="${PROJECT_DIR}/dist" + +echo "=== TundraDB Linux Bundle Creator ===" +echo "Version : ${VERSION}" +echo "Image : ${IMAGE_NAME}" +echo "Project : ${PROJECT_DIR}" +echo "Output : ${DIST_DIR}/" +echo "" + +# --- 1. Check prerequisites --- +if ! command -v docker &>/dev/null; then + echo "❌ Docker not found. Please install Docker Desktop." + exit 1 +fi + +if ! docker info &>/dev/null 2>&1; then + echo "❌ Docker daemon is not running. Please start Docker Desktop." + exit 1 +fi + +# --- 2. Build the Docker image --- +if [ "$SKIP_BUILD" = false ]; then + echo ">>> Building Docker image (this will take a few minutes on first run)..." + echo " Subsequent builds use Docker layer caching and are much faster." + echo "" + + docker build \ + --platform linux/amd64 \ + -f "${PROJECT_DIR}/docker/Dockerfile.linux-release" \ + -t "${IMAGE_NAME}" \ + --build-arg TUNDRADB_VERSION="${VERSION}" \ + ${NO_CACHE} \ + "${PROJECT_DIR}" + + echo "" + echo "✅ Docker image built successfully." +else + echo ">>> Skipping build (--skip-build). Reusing existing image." + if ! docker image inspect "${IMAGE_NAME}" &>/dev/null; then + echo "❌ Image '${IMAGE_NAME}' not found. Run without --skip-build first." + exit 1 + fi +fi + +# --- 3. Extract the tarball --- +echo "" +echo ">>> Extracting Linux tarball from Docker image..." + +mkdir -p "${DIST_DIR}" + +# Remove any previous extraction container +docker rm -f "${CONTAINER_NAME}" &>/dev/null || true + +# Create a container (don't start it) and copy the tarball out +docker create --platform linux/amd64 --name "${CONTAINER_NAME}" "${IMAGE_NAME}" /bin/true +docker cp "${CONTAINER_NAME}:/dist/TundraDB-${VERSION}-Linux-x86_64.tar.gz" "${DIST_DIR}/" +docker rm "${CONTAINER_NAME}" + +ARCHIVE="${DIST_DIR}/TundraDB-${VERSION}-Linux-x86_64.tar.gz" +ARCHIVE_SIZE=$(du -sh "${ARCHIVE}" | awk '{print $1}') + +echo "" +echo "=== Done ===" +echo "📦 Archive : dist/TundraDB-${VERSION}-Linux-x86_64.tar.gz (${ARCHIVE_SIZE})" +echo "" +echo "To use on a Linux machine:" +echo " scp dist/TundraDB-${VERSION}-Linux-x86_64.tar.gz user@host:~/" +echo " ssh user@host" +echo " tar xzf TundraDB-${VERSION}-Linux-x86_64.tar.gz" +echo " cd TundraDB-${VERSION}-Linux-x86_64" +echo " ./tundra_shell --db-path ./my-database" +