diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index b4bd6dc20..64c635f27 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,21 +1,18 @@ -FROM --platform=linux/amd64 archlinux@sha256:c84ad63503efc5d386e30a5f44945fa8eee10fcb21b3d5fceda260783de394ff +FROM archlinux@sha256:c84ad63503efc5d386e30a5f44945fa8eee10fcb21b3d5fceda260783de394ff LABEL maintainer="Jguer,docker@jguer.space" ENV GO111MODULE=on WORKDIR /app -COPY go.mod . - # Disable pacman's landlock/seccomp sandbox: it doesn't work inside BuildKit. # asciidoc, doxygen, meson needed for pacman-git. RUN set -eux; \ sed -i 's/^#\?DisableSandbox.*/DisableSandbox/' /etc/pacman.conf || true; \ sed -i 's/^#\?DownloadUser.*/#DownloadUser = alpm/' /etc/pacman.conf || true; \ pacman -Syu --noconfirm --needed \ - pacman-contrib zsh git-delta openssh bat go github-cli ripgrep \ + pacman-contrib zsh git-delta openssh bat go github-cli ripgrep pre-commit \ archlinux-keyring git gcc make sudo asciidoc doxygen meson curl nodejs npm; \ curl -sSfL https://golangci-lint.run/install.sh | sh -s v2.12.2; \ - go mod download; \ rm -rf /var/lib/pacman/sync/* /var/cache/pacman/* /tmp/* /var/tmp/*; \ rm -rf /usr/share/man/* /usr/share/doc/* || true @@ -24,8 +21,7 @@ RUN useradd -m -s /bin/bash docker \ USER docker -RUN go install github.com/leonelquinteros/gotext/cli/xgotext@latest \ - && go install golang.org/x/tools/gopls@latest +RUN go install github.com/leonelquinteros/gotext/cli/xgotext@v1.7.2 ENV PATH="/app/bin:/home/docker/go/bin:/home/docker/.local/bin:$PATH" diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 24bd7dd38..4af200bfd 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -48,4 +48,33 @@ jobs: name: yay path: ./yay if-no-files-found: error - overwrite: true \ No newline at end of file + overwrite: true + + e2e: + name: E2E PKGBUILD repositories + runs-on: ubuntu-latest + container: + image: ghcr.io/jguer/yay-builder:latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0 + with: + go-version-file: go.mod + - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + # makepkg refuses to run as root, so the e2e suite runs as an + # unprivileged user with passwordless sudo for pacman. + - name: Set up unprivileged build user + run: | + pacman -Sy --needed --noconfirm base-devel git + id -u e2e >/dev/null 2>&1 || useradd -m e2e + echo 'e2e ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/e2e + chmod 0440 /etc/sudoers.d/e2e + chown -R e2e:e2e . + install -d -o e2e -g e2e /home/e2e/go + - name: Run E2E tests + run: su e2e -c "env HOME=/home/e2e GOFLAGS=-buildvcs=false make test-e2e" diff --git a/Makefile b/Makefile index 23d3342df..17346e623 100644 --- a/Makefile +++ b/Makefile @@ -56,6 +56,13 @@ test: test-integration: $(GO) test -tags=integration $(FLAGS) ./... +# End-to-end tests for PKGBUILD repository support. Requires an Arch Linux +# environment with makepkg/pacman/git and passwordless sudo, run as a non-root +# user (makepkg refuses to run as root). See e2e/README.md. +.PHONY: test-e2e +test-e2e: + ./e2e/run.sh + .PHONY: build build: $(BIN) diff --git a/doc/lua.md b/doc/lua.md index 632db98e8..6798428ac 100644 --- a/doc/lua.md +++ b/doc/lua.md @@ -89,6 +89,51 @@ startup and reports the offending keys/values so misconfigurations fail fast. A ready-to-copy example lives at [`doc/init.lua`](init.lua). +## PKGBUILD repositories with `yay.opt.pkgbuild_repos` + +A PKGBUILD repository is a directory of PKGBUILDs — local or served over git — +that yay treats much like the AUR. Packages found in a PKGBUILD repository +**take priority over the AUR**, so a repository can mask an AUR package with a +custom build. Installed packages owned by a repository are upgraded from it on +`yay -Syu` (or `yay -Sua`) instead of the AUR. PKGBUILD repositories are +configured only through `init.lua`. + +Assign a table keyed by repository name. Each entry has a `url` and an optional +`depth`: + +```lua +yay.opt.pkgbuild_repos = { + ["yay-pkgbuild"] = { + url = "https://github.com/Jguer/yay-PKGBUILD", + depth = 3, + }, + -- A local git repository: + ["local-git"] = { url = "git+file:///srv/pkgbuild-repo" }, + -- A plain local directory, used in place with no clone or refresh: + ["scratch"] = { url = "file:///home/user/pkgbuilds" }, +} +``` + +### `url` + +The repository location, following the makepkg source convention. The scheme +decides how yay treats it: + +- `https://`, `ssh://`, a `.git` suffix, or a `git+…` prefix are cloned into + yay's build cache. The clone is created on first use and pulled again on + `yay -Sy`, so routine installs work offline. Use authenticated transports; + insecure `http://` and `git://` URLs are not supported. +- `file://…` points at a local directory that is scanned in place — no clone, + no refresh. + +### `depth` + +How many directory levels below the repository root yay scans for PKGBUILDs. +Defaults to `3`. A repository is a directory tree of package directories, each +containing a `PKGBUILD` and a committed `.SRCINFO`. yay never generates +`.SRCINFO` while indexing: `makepkg --printsrcinfo` executes PKGBUILD content, +so metadata generation belongs in the repository's reviewed build workflow. + ## Logging with `yay.log`

Available from yay v13.0.0

@@ -405,7 +450,7 @@ logs the message but cannot roll back anything. name = "pkgname", version = "1.2.3-1", -- resolved version local_version = "1.0.0-1", -- previously installed ("" if not installed) - source = "aur", -- "aur" | "sync" | "local" | "srcinfo" | "missing" + source = "aur", -- "aur" | "sync" | "local" | "srcinfo" | "pkgbuild_repo" | "missing" reason = "explicit", -- "explicit" | "dependency" | "make_dependency" | "check_dependency" | "unknown" }, -- one entry per package yay resolved; sorted alphabetically diff --git a/e2e/README.md b/e2e/README.md new file mode 100644 index 000000000..0929ffad3 --- /dev/null +++ b/e2e/README.md @@ -0,0 +1,33 @@ +# End-to-end tests + +`run.sh` exercises PKGBUILD repository support (`yay.opt.pkgbuild_repos`) against +real `makepkg` and `pacman`, rather than mocks. It builds `yay` and then: + +1. **`file://` repository** — installs a package from a local directory + repository used in place. +2. **`-Sua` upgrade** — bumps that package and upgrades it from the repository. +3. **`git+file://` repository** — installs a package from a local git + repository, which yay clones into its cache. + +Each package is trivial (`arch=('any')`, no sources) and does not exist in the +AUR, so a successful install proves the package was resolved from the configured +repository. The test packages (`yay-e2e-file`, `yay-e2e-git`) are removed on +exit. + +## Requirements + +- An Arch Linux environment with `makepkg`, `pacman`, `fakeroot`, `git`. +- Passwordless `sudo` (yay runs `sudo pacman` to install built packages). +- A **non-root** user — `makepkg` refuses to run as root. + +## Running + +```sh +make test-e2e +# or, against an existing binary: +YAY_BIN=/path/to/yay ./e2e/run.sh +``` + +CI runs this via the `E2E PKGBUILD repositories` job in +`.github/workflows/testing.yml`, using the Arch-based `yay-builder` image and an +unprivileged user. diff --git a/e2e/fixtures/PKGBUILD.in b/e2e/fixtures/PKGBUILD.in new file mode 100644 index 000000000..3b663f688 --- /dev/null +++ b/e2e/fixtures/PKGBUILD.in @@ -0,0 +1,14 @@ +# Template PKGBUILD for yay PKGBUILD-repository e2e tests. +# @PKGNAME@ and @PKGVER@ are substituted by e2e/run.sh. The package builds +# instantly (no sources, no compilation) and just drops a marker file so a +# successful install proves the package came from the configured repository. +pkgname=@PKGNAME@ +pkgver=@PKGVER@ +pkgrel=1 +pkgdesc="yay e2e PKGBUILD repository test package" +arch=('any') +license=('MIT') + +package() { + install -Dm644 /dev/null "$pkgdir/usr/share/@PKGNAME@/marker" +} diff --git a/e2e/run.sh b/e2e/run.sh new file mode 100755 index 000000000..72e715f5f --- /dev/null +++ b/e2e/run.sh @@ -0,0 +1,146 @@ +#!/usr/bin/env bash +# +# End-to-end tests for PKGBUILD repository support (yay.opt.pkgbuild_repos). +# +# Builds yay, then drives real makepkg/pacman installs and upgrades from local +# file:// and git+file:// PKGBUILD repositories, asserting the results. A +# successful install proves the package was resolved from the configured +# repository (the test packages do not exist in the AUR). +# +# Requirements: an Arch Linux environment with makepkg, pacman, git and +# passwordless sudo, run as a NON-root user (makepkg refuses to run as root). +# +# Usage: +# e2e/run.sh # builds yay itself +# YAY_BIN=/path/to/yay e2e/run.sh # uses an existing binary +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +readonly REPO_ROOT +readonly FIXTURES="${REPO_ROOT}/e2e/fixtures" + +GO="${GO:-go}" +YAY_BIN="${YAY_BIN:-}" + +fail() { + echo "E2E FAIL: $*" >&2 + exit 1 +} + +info() { + echo "==> $*" +} + +# --- preconditions -------------------------------------------------------- + +[[ ${EUID} -ne 0 ]] || fail "must run as a non-root user (makepkg refuses to run as root)" + +for tool in makepkg pacman fakeroot git sudo; do + command -v "${tool}" >/dev/null 2>&1 || fail "missing required tool: ${tool}" +done + +sudo -n true 2>/dev/null || fail "passwordless sudo is required" + +# --- workspace ------------------------------------------------------------ + +WORK="$(mktemp -d)" +readonly WORK +INSTALLED=() + +cleanup() { + if [[ ${#INSTALLED[@]} -gt 0 ]]; then + sudo pacman -R --noconfirm "${INSTALLED[@]}" >/dev/null 2>&1 || true + fi + rm -rf "${WORK}" +} +trap cleanup EXIT + +if [[ -z "${YAY_BIN}" ]]; then + info "building yay" + YAY_BIN="${WORK}/yay" + (cd "${REPO_ROOT}" && "${GO}" build -o "${YAY_BIN}" .) +fi +readonly YAY_BIN + +# Isolate config and cache so the host environment is untouched. +export XDG_CONFIG_HOME="${WORK}/config" +export XDG_CACHE_HOME="${WORK}/cache" +mkdir -p "${XDG_CONFIG_HOME}/yay" + +# --- helpers -------------------------------------------------------------- + +# make_pkg +# Instantiates the fixture PKGBUILD into repo_dir/pkgname with a fresh .SRCINFO, +# clearing any previous build artifacts so upgrades rebuild cleanly. +make_pkg() { + local repo_dir="$1" name="$2" version="$3" + local pkg_dir="${repo_dir}/${name}" + + mkdir -p "${pkg_dir}" + rm -rf "${pkg_dir}"/pkg "${pkg_dir}"/src "${pkg_dir}"/*.pkg.tar* 2>/dev/null || true + sed -e "s/@PKGNAME@/${name}/g" -e "s/@PKGVER@/${version}/g" \ + "${FIXTURES}/PKGBUILD.in" >"${pkg_dir}/PKGBUILD" + (cd "${pkg_dir}" && makepkg --printsrcinfo >.SRCINFO) +} + +# write_config +write_config() { + cat >"${XDG_CONFIG_HOME}/yay/init.lua" < +assert_version() { + local name="$1" want="$2" got + got="$(pacman -Q "${name}" 2>/dev/null | awk '{print $2}')" || fail "${name} is not installed" + [[ "${got}" == "${want}" ]] || fail "${name}: expected version ${want}, got ${got}" + info "OK: ${name} ${got}" +} + +# --- scenario 1: install from a file:// local directory repository -------- + +info "scenario 1: install from a file:// local directory repository" +FILE_REPO="${WORK}/file-repo" +make_pkg "${FILE_REPO}" "yay-e2e-file" "1" +write_config "file-repo" "file://${FILE_REPO}" + +sudo pacman -R --noconfirm yay-e2e-file >/dev/null 2>&1 || true +"${YAY_BIN}" -S yay-e2e-file --noconfirm +INSTALLED+=("yay-e2e-file") +assert_version yay-e2e-file "1-1" + +# --- scenario 2: upgrade a repo package with -Sua ------------------------- + +info "scenario 2: upgrade a PKGBUILD-repo package with -Sua" +make_pkg "${FILE_REPO}" "yay-e2e-file" "2" +"${YAY_BIN}" -Sua --noconfirm +assert_version yay-e2e-file "2-1" + +# --- scenario 3: install from a git+file:// repository -------------------- + +info "scenario 3: install from a git+file:// repository" +GIT_REPO="${WORK}/git-repo" +make_pkg "${GIT_REPO}" "yay-e2e-git" "1" +git -C "${GIT_REPO}" init -q +git -C "${GIT_REPO}" config user.email "e2e@example.invalid" +git -C "${GIT_REPO}" config user.name "yay e2e" +git -C "${GIT_REPO}" config commit.gpgsign false +git -C "${GIT_REPO}" add -A +git -C "${GIT_REPO}" commit -qm "init" + +write_config "git-repo" "git+file://${GIT_REPO}" + +sudo pacman -R --noconfirm yay-e2e-git >/dev/null 2>&1 || true +"${YAY_BIN}" -S yay-e2e-git --noconfirm +INSTALLED+=("yay-e2e-git") +assert_version yay-e2e-git "1-1" + +[[ -d "${XDG_CACHE_HOME}/yay/.pkgbuild-repos/git-repo/.git" ]] || + fail "git repository was not cloned into the cache" +info "OK: git repository cloned into cache" + +echo "E2E PASS: all PKGBUILD repository scenarios succeeded" diff --git a/local_install.go b/local_install.go index f9fd927ab..4649e3886 100644 --- a/local_install.go +++ b/local_install.go @@ -86,6 +86,13 @@ func installLocalPKGBUILD( grapher := dep.NewGrapher(dbExecutor, aurCache, false, settings.NoConfirm, cmdArgs.ExistsDouble("d", "nodeps"), noCheck, cmdArgs.ExistsArg("needed"), run.Logger.Child("grapher")) + + pkgbuildRepos, err := loadPkgbuildRepoIndex(ctx, run, false) + if err != nil { + return fmt.Errorf("%s: %w", gotext.Get("error loading PKGBUILD repositories"), err) + } + grapher.SetPkgbuildRepos(pkgbuildRepos) + graph, err := grapher.GraphFromSrcInfos(ctx, nil, srcInfos) if err != nil { return err diff --git a/main.go b/main.go index d888f59e4..f4d98df17 100644 --- a/main.go +++ b/main.go @@ -95,6 +95,13 @@ func main() { defer luaEngine.Close() } + if err = cfg.NormalizePkgbuildRepos(); err != nil { + fallbackLog.Errorln(err) + ret = 1 + + return + } + cmdArgs := parser.MakeArguments() // Parse command line diff --git a/meta/yay.d.lua b/meta/yay.d.lua index 7766410bd..24bf8c7a7 100644 --- a/meta/yay.d.lua +++ b/meta/yay.d.lua @@ -63,6 +63,13 @@ ---@field debug boolean Enable debug logging and local init.lua lookup convenience. ---@field rpc boolean Use AUR RPC for dependency/query operations. ---@field double_confirm boolean Ask for confirmation before and after builds during upgrades. +---@field pkgbuild_repos table Named PKGBUILD repositories that take priority over the AUR. init.lua only. + +-- PKGBUILD repositories: yay.opt.pkgbuild_repos + +---@class yay.PkgbuildRepo +---@field url string Repo location, following the makepkg source convention: an https git URL, a git+file:// local git repo, or a file:// local directory used in place. +---@field depth? integer Recursive PKGBUILD scan depth (default 3). -- Logging: yay.log diff --git a/pkg/dep/dep_graph.go b/pkg/dep/dep_graph.go index b46731fac..c337074a3 100644 --- a/pkg/dep/dep_graph.go +++ b/pkg/dep/dep_graph.go @@ -17,6 +17,7 @@ import ( "github.com/Jguer/yay/v13/pkg/db" "github.com/Jguer/yay/v13/pkg/dep/topo" "github.com/Jguer/yay/v13/pkg/intrange" + "github.com/Jguer/yay/v13/pkg/pkgbuildrepo" aur "github.com/Jguer/yay/v13/pkg/query" "github.com/Jguer/yay/v13/pkg/text" ) @@ -27,6 +28,7 @@ type InstallInfo struct { LocalVersion string AURBase string SyncDBName string + RepoName string // PKGBUILD-repo name this package is attributed to, for display. SrcinfoPath string Maintainer string Source Source @@ -53,6 +55,12 @@ func (s Source) String() string { return SourceNames[s] } +// IsBuiltFromSource reports whether packages from this source are built locally +// through the makepkg pipeline, rather than installed by pacman from a sync DB. +func (s Source) IsBuiltFromSource() bool { + return s == AUR || s == SrcInfo || s == PkgbuildRepo +} + const ( Explicit Reason = iota // 0 Dep // 1 @@ -72,22 +80,25 @@ const ( Sync Local SrcInfo + PkgbuildRepo Missing ) var SourceNames = map[Source]string{ - AUR: gotext.Get("AUR"), - Sync: gotext.Get("Sync"), - Local: gotext.Get("Local"), - SrcInfo: gotext.Get("SRCINFO"), - Missing: gotext.Get("Missing"), + AUR: gotext.Get("AUR"), + Sync: gotext.Get("Sync"), + Local: gotext.Get("Local"), + SrcInfo: gotext.Get("SRCINFO"), + PkgbuildRepo: gotext.Get("PKGBUILD Repo"), + Missing: gotext.Get("Missing"), } var bgColorMap = map[Source]string{ - AUR: "lightblue", - Sync: "lemonchiffon", - Local: "darkolivegreen1", - Missing: "tomato", + AUR: "lightblue", + Sync: "lemonchiffon", + Local: "darkolivegreen1", + PkgbuildRepo: "lightcyan", + Missing: "tomato", } var colorMap = map[Reason]string{ @@ -101,13 +112,40 @@ type Grapher struct { logger *text.Logger providerCache map[string][]aur.Pkg - dbExecutor db.Executor - aurClient aurc.QueryClient - fullGraph bool // If true, the graph will include all dependencies including already installed ones or repo - noConfirm bool // If true, the graph will not prompt for confirmation - noDeps bool // If true, the graph will not include dependencies - noCheckDeps bool // If true, the graph will not include check dependencies - needed bool // If true, the graph will only include packages that are not installed + dbExecutor db.Executor + aurClient aurc.QueryClient + + // pkgbuildRepo holds the configured PKGBUILD repos (masks the AUR); may be nil. + pkgbuildRepo *pkgbuildrepo.Index + // repoPkgCache memoizes the srcinfo->pkg derivation per repo entry. + repoPkgCache map[*pkgbuildrepo.Entry][]*aurc.Pkg + + fullGraph bool // include all dependencies, including already-installed or repo ones + noConfirm bool // do not prompt for confirmation + noDeps bool // do not include dependencies + noCheckDeps bool // do not include check dependencies + needed bool // only include packages that are not installed +} + +// SetPkgbuildRepos wires the configured PKGBUILD repositories into the grapher. +// Packages found there mask AUR packages during resolution. +func (g *Grapher) SetPkgbuildRepos(index *pkgbuildrepo.Index) { + g.pkgbuildRepo = index +} + +// HasPkgbuildRepos reports whether any PKGBUILD repositories are configured. +func (g *Grapher) HasPkgbuildRepos() bool { + return g.pkgbuildRepo != nil +} + +// PkgbuildRepoEntry returns the repo entry providing name, if any repo is +// configured and contains it. +func (g *Grapher) PkgbuildRepoEntry(name string) (*pkgbuildrepo.Entry, bool) { + if g.pkgbuildRepo == nil { + return nil, false + } + + return g.pkgbuildRepo.Get(name) } func NewGrapher(dbExecutor db.Executor, aurCache aurc.QueryClient, @@ -123,6 +161,7 @@ func NewGrapher(dbExecutor db.Executor, aurCache aurc.QueryClient, noCheckDeps: noCheckDeps, needed: needed, providerCache: make(map[string][]aurc.Pkg, 5), + repoPkgCache: make(map[*pkgbuildrepo.Entry][]*aurc.Pkg), logger: logger, } } @@ -159,6 +198,14 @@ func (g *Grapher) GraphFromTargets(ctx context.Context, continue } + if entry, ok := g.PkgbuildRepoEntry(target.Name); ok { + if err := g.graphPkgbuildRepoEntry(ctx, graph, entry, Explicit); err != nil { + return nil, err + } + + continue + } + fallthrough case "aur": aurTargets = append(aurTargets, target.Name) @@ -292,6 +339,137 @@ func (g *Grapher) GraphFromSrcInfos(ctx context.Context, graph *topo.Graph[strin return graph, nil } +// repoEntryPkgs derives (and memoizes) the aur.Pkgs of a PKGBUILD-repo entry. +// entry.Srcinfo is immutable, so the derivation is cached per entry to avoid +// repeated AlpmArchitectures calls and allocations across references. +func (g *Grapher) repoEntryPkgs(entry *pkgbuildrepo.Entry) ([]*aurc.Pkg, error) { + if pkgs, ok := g.repoPkgCache[entry]; ok { + return pkgs, nil + } + + pkgs, err := makeAURPKGFromSrcinfo(g.dbExecutor, entry.Srcinfo) + if err != nil { + return nil, err + } + + g.repoPkgCache[entry] = pkgs + + return pkgs, nil +} + +// repoInstallInfo builds the InstallInfo shared by every PKGBUILD-repo package +// node. Callers set the deltas (upgrade fields) on the returned value. +func repoInstallInfo(entry *pkgbuildrepo.Entry, pkg *aurc.Pkg, reason Reason) *InstallInfo { + return &InstallInfo{ + Source: PkgbuildRepo, + Reason: reason, + SrcinfoPath: entry.Dir, + AURBase: pkg.PackageBase, + RepoName: entry.RepoName, + Version: pkg.Version, + } +} + +// graphPkgbuildRepoPkg adds one PKGBUILD-repo package node (with its provides +// and dependencies) built from the repo's local PKGBUILD directory. +func (g *Grapher) graphPkgbuildRepoPkg(ctx context.Context, + graph *topo.Graph[string, *InstallInfo], pkg *aurc.Pkg, info *InstallInfo, +) { + graph.AddNode(pkg.Name) + g.addAurPkgProvides(pkg, graph) + + g.ValidateAndSetNodeInfo(graph, pkg.Name, &topo.NodeInfo[*InstallInfo]{ + Color: colorMap[info.Reason], + Background: bgColorMap[PkgbuildRepo], + Value: info, + }) + + g.addDepNodes(ctx, pkg, graph) +} + +// graphPkgbuildRepoEntry graphs every package of a PKGBUILD-repo base as a +// target, lowering the reason to match an already-installed package. +func (g *Grapher) graphPkgbuildRepoEntry(ctx context.Context, + graph *topo.Graph[string, *InstallInfo], entry *pkgbuildrepo.Entry, reason Reason, +) error { + aurPkgs, err := g.repoEntryPkgs(entry) + if err != nil { + return err + } + + for _, pkg := range aurPkgs { + g.graphPkgbuildRepoPkg(ctx, graph, pkg, repoInstallInfo(entry, pkg, g.localReason(pkg.Name, reason))) + } + + return nil +} + +// graphPkgbuildRepoDep graphs the single package in entry that satisfies dep. +// Dependencies must resolve to a concrete package node: virtual provides are +// metadata, not packages that can be built or installed. +func (g *Grapher) graphPkgbuildRepoDep(ctx context.Context, + graph *topo.Graph[string, *InstallInfo], entry *pkgbuildrepo.Entry, dep string, reason Reason, +) (string, error) { + aurPkgs, err := g.repoEntryPkgs(entry) + if err != nil { + return "", err + } + + for _, pkg := range aurPkgs { + if !satisfiesAur(dep, pkg) { + continue + } + + g.graphPkgbuildRepoPkg(ctx, graph, pkg, repoInstallInfo(entry, pkg, g.localReason(pkg.Name, reason))) + + return pkg.Name, nil + } + + return "", nil +} + +// GraphPkgbuildRepoUpgrade graphs an installed package that a configured +// PKGBUILD repo can upgrade, marking the node as an upgrade and adding its +// dependencies. name is the installed package (a pkgname of entry's base). +func (g *Grapher) GraphPkgbuildRepoUpgrade(ctx context.Context, + graph *topo.Graph[string, *InstallInfo], entry *pkgbuildrepo.Entry, + name, localVersion string, reason Reason, +) (*topo.Graph[string, *InstallInfo], error) { + if graph == nil { + graph = NewGraph() + } + + aurPkgs, err := g.repoEntryPkgs(entry) + if err != nil { + return graph, err + } + + for _, pkg := range aurPkgs { + if pkg.Name != name { + continue + } + + info := repoInstallInfo(entry, pkg, reason) + info.LocalVersion = localVersion + info.Upgrade = true + g.graphPkgbuildRepoPkg(ctx, graph, pkg, info) + } + + return graph, nil +} + +// localReason lowers reason to the install reason of name if it is already +// installed (an installed explicit package must not be downgraded to a dep). +func (g *Grapher) localReason(name string, reason Reason) Reason { + if localPkg := g.dbExecutor.LocalPackage(name); localPkg != nil { + if r := Reason(localPkg.Reason()); r < reason { + return r + } + } + + return reason +} + func (g *Grapher) AddDepsForPkgs(ctx context.Context, pkgs []*aur.Pkg, graph *topo.Graph[string, *InstallInfo]) { for _, pkg := range pkgs { g.addDepNodes(ctx, pkg, graph) @@ -717,6 +895,37 @@ func (g *Grapher) addNodes( } pending = keep + // Check PKGBUILD repos (mask the AUR) + keep = pending[:0] + for _, depString := range pending { + depName, _, _ := splitDep(depString) + + entry, ok := g.PkgbuildRepoEntry(depName) + if !ok { + keep = append(keep, depString) + + continue + } + + pkgName, err := g.graphPkgbuildRepoDep(ctx, graph, entry, depString, depType) + if err != nil { + g.logger.Warnln("pkgbuild repo dep warn:", depString, parentPkgName, err) + keep = append(keep, depString) + + continue + } + if pkgName == "" { + keep = append(keep, depString) + + continue + } + + if err := graph.DependOn(pkgName, parentPkgName); err != nil { + g.logger.Warnln("pkgbuild repo dep warn:", depString, parentPkgName, err) + } + } + pending = keep + // Check AUR pkgsToAdd, missing := g.findDepsFromAUR(ctx, graph, parentPkgName, pending) for i := range pkgsToAdd { diff --git a/pkg/dep/dep_pkgbuild_repo_test.go b/pkg/dep/dep_pkgbuild_repo_test.go new file mode 100644 index 000000000..9060a571c --- /dev/null +++ b/pkg/dep/dep_pkgbuild_repo_test.go @@ -0,0 +1,177 @@ +//go:build !integration + +package dep + +import ( + "context" + "io" + "os" + "path/filepath" + "strings" + "testing" + + aurc "github.com/Jguer/aur" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/Jguer/yay/v13/pkg/db/mock" + mockaur "github.com/Jguer/yay/v13/pkg/dep/mock" + "github.com/Jguer/yay/v13/pkg/pkgbuildrepo" + aur "github.com/Jguer/yay/v13/pkg/query" + "github.com/Jguer/yay/v13/pkg/text" +) + +func newFooRepoIndex(t *testing.T) (idx *pkgbuildrepo.Index, pkgDir string) { + t.Helper() + + pkgDir = filepath.Join(t.TempDir(), "foo") + require.NoError(t, os.MkdirAll(pkgDir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(pkgDir, ".SRCINFO"), + []byte("pkgbase = foo\n\tpkgver = 1\n\tpkgrel = 1\n\tarch = x86_64\n\npkgname = foo\n"), 0o600)) + + idx = pkgbuildrepo.NewIndex() + require.NoError(t, idx.AddRepo("myrepo", []string{pkgDir})) + + return idx, pkgDir +} + +func writeRepoSrcinfo(t *testing.T, dir, contents string) { + t.Helper() + require.NoError(t, os.MkdirAll(dir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, ".SRCINFO"), []byte(contents), 0o600)) +} + +// GIVEN a target present in a PKGBUILD repo +// WHEN it is graphed +// THEN it resolves to the PkgbuildRepo source pointing at the repo dir, and the +// AUR is never queried for it. +func TestGrapher_GraphFromTargets_pkgbuildRepoMasksAUR(t *testing.T) { + t.Parallel() + + idx, pkgDir := newFooRepoIndex(t) + + mockDB := &mock.DBExecutor{ + AlpmArchitecturesFn: func() ([]string, error) { return []string{"x86_64"}, nil }, + SyncSatisfierFn: func(string) mock.IPackage { return nil }, + PackagesFromGroupFn: func(string) []mock.IPackage { return nil }, + LocalPackageFn: func(string) mock.IPackage { return nil }, + } + mockAUR := &mockaur.MockAUR{GetFn: func(_ context.Context, query *aurc.Query) ([]aur.Pkg, error) { + t.Errorf("AUR must not be queried for a pkgbuild-repo package, got %+v", query) + return nil, nil + }} + + logger := text.NewLogger(io.Discard, io.Discard, strings.NewReader(""), false, "test") + g := NewGrapher(mockDB, mockAUR, false, true, true, true, false, logger) + g.SetPkgbuildRepos(idx) + + graph, err := g.GraphFromTargets(t.Context(), nil, []string{"foo"}) + require.NoError(t, err) + + info := graph.GetNodeInfo("foo") + require.NotNil(t, info) + require.NotNil(t, info.Value) + assert.Equal(t, PkgbuildRepo, info.Value.Source) + assert.Equal(t, pkgDir, info.Value.SrcinfoPath) + assert.Equal(t, "foo", info.Value.AURBase) +} + +// GIVEN a dependency satisfiable by a PKGBUILD repo +// WHEN a package depending on it is graphed +// THEN the dependency resolves to the PkgbuildRepo source, not the AUR. +func TestGrapher_addNodes_pkgbuildRepoDep(t *testing.T) { + t.Parallel() + + idx, pkgDir := newFooRepoIndex(t) + + mockDB := &mock.DBExecutor{ + AlpmArchitecturesFn: func() ([]string, error) { return []string{"x86_64"}, nil }, + SyncSatisfierFn: func(string) mock.IPackage { return nil }, + LocalSatisfierExistsFn: func(string) bool { return false }, + LocalPackageFn: func(string) mock.IPackage { return nil }, + } + mockAUR := &mockaur.MockAUR{GetFn: func(_ context.Context, query *aurc.Query) ([]aur.Pkg, error) { + t.Errorf("AUR must not be queried for a pkgbuild-repo dependency, got %+v", query) + return nil, nil + }} + + logger := text.NewLogger(io.Discard, io.Discard, strings.NewReader(""), false, "test") + g := NewGrapher(mockDB, mockAUR, false, true, false, false, false, logger) + g.SetPkgbuildRepos(idx) + + graph := NewGraph() + graph.AddNode("parent") + g.addNodes(t.Context(), graph, "parent", []string{"foo>=1"}, Dep) + + info := graph.GetNodeInfo("foo") + require.NotNil(t, info) + require.NotNil(t, info.Value) + assert.Equal(t, PkgbuildRepo, info.Value.Source) + assert.Equal(t, pkgDir, info.Value.SrcinfoPath) +} + +func TestGrapher_graphPkgbuildRepoDepSelectsConcreteSatisfier(t *testing.T) { + t.Parallel() + + pkgDir := filepath.Join(t.TempDir(), "provider") + writeRepoSrcinfo(t, pkgDir, "pkgbase = provider\n\tpkgver = 1\n\tpkgrel = 1\n\tarch = x86_64\n\npkgname = provider\n\tprovides = virtual=1\n") + idx := pkgbuildrepo.NewIndex() + require.NoError(t, idx.AddRepo("myrepo", []string{pkgDir})) + + dbExe := &mock.DBExecutor{ + AlpmArchitecturesFn: func() ([]string, error) { return []string{"x86_64"}, nil }, + LocalPackageFn: func(string) mock.IPackage { return nil }, + } + logger := text.NewLogger(io.Discard, io.Discard, strings.NewReader(""), false, "test") + g := NewGrapher(dbExe, &mockaur.MockAUR{}, false, true, false, false, false, logger) + entry, ok := idx.Get("virtual") + require.True(t, ok) + + graph := NewGraph() + pkgName, err := g.graphPkgbuildRepoDep(t.Context(), graph, entry, "virtual>=1", Dep) + require.NoError(t, err) + assert.Equal(t, "provider", pkgName) + assert.NotNil(t, graph.GetNodeInfo("provider").Value) + assert.Nil(t, graph.GetNodeInfo("virtual")) + + pkgName, err = g.graphPkgbuildRepoDep(t.Context(), graph, entry, "virtual>=2", Dep) + require.NoError(t, err) + assert.Empty(t, pkgName) +} + +// GIVEN an installed package a repo has a newer version of +// WHEN graphed as an upgrade +// THEN the node is marked as an upgrade with the local and remote versions. +func TestGrapher_GraphPkgbuildRepoUpgrade(t *testing.T) { + t.Parallel() + + idx, pkgDir := newFooRepoIndex(t) + entry, ok := idx.Get("foo") + require.True(t, ok) + + mockDB := &mock.DBExecutor{ + AlpmArchitecturesFn: func() ([]string, error) { return []string{"x86_64"}, nil }, + LocalSatisfierExistsFn: func(string) bool { return false }, + } + mockAUR := &mockaur.MockAUR{GetFn: func(_ context.Context, query *aurc.Query) ([]aur.Pkg, error) { + t.Errorf("AUR must not be queried, got %+v", query) + return nil, nil + }} + + logger := text.NewLogger(io.Discard, io.Discard, strings.NewReader(""), false, "test") + g := NewGrapher(mockDB, mockAUR, false, true, false, false, false, logger) + g.SetPkgbuildRepos(idx) + + graph, err := g.GraphPkgbuildRepoUpgrade(t.Context(), NewGraph(), entry, "foo", "0.9-1", Dep) + require.NoError(t, err) + + info := graph.GetNodeInfo("foo") + require.NotNil(t, info) + require.NotNil(t, info.Value) + assert.Equal(t, PkgbuildRepo, info.Value.Source) + assert.True(t, info.Value.Upgrade) + assert.Equal(t, "0.9-1", info.Value.LocalVersion) + assert.Equal(t, "1-1", info.Value.Version) + assert.Equal(t, pkgDir, info.Value.SrcinfoPath) + assert.Equal(t, "myrepo", info.Value.RepoName) +} diff --git a/pkg/download/unified.go b/pkg/download/unified.go index 90d9e412c..b12960598 100644 --- a/pkg/download/unified.go +++ b/pkg/download/unified.go @@ -73,6 +73,14 @@ func downloadGitRepo(ctx context.Context, cmdBuilder exe.GitCmdBuilder, return newClone, nil } +// PkgbuildRepoClone clones or updates a PKGBUILD repository's git repo into +// dest/name, returning whether it was a fresh clone. +func PkgbuildRepoClone(ctx context.Context, cmdBuilder exe.GitCmdBuilder, + cloneURL, name, dest string, force bool, +) (bool, error) { + return downloadGitRepo(ctx, cmdBuilder, cloneURL, name, dest, force) +} + func getURLName(pkg db.IPackage) string { name := pkg.Base() if name == "" { diff --git a/pkg/pkgbuildrepo/index.go b/pkg/pkgbuildrepo/index.go new file mode 100644 index 000000000..9940990f0 --- /dev/null +++ b/pkg/pkgbuildrepo/index.go @@ -0,0 +1,106 @@ +package pkgbuildrepo + +import ( + "fmt" + "path/filepath" + "strings" + + gosrc "github.com/Morganamilo/go-srcinfo" +) + +// Entry is one PKGBUILD found in a repository, keyed by its package base. +type Entry struct { + RepoName string + Dir string + Pkgbase string + Version string + Srcinfo *gosrc.Srcinfo +} + +// Index resolves package names to the PKGBUILD-repo entry that provides them. +// Real package names (pkgbase/pkgname) always take priority over provides, +// regardless of the order repos are added. +type Index struct { + byName map[string]*Entry + byProvides map[string]*Entry +} + +func NewIndex() *Index { + return &Index{ + byName: map[string]*Entry{}, + byProvides: map[string]*Entry{}, + } +} + +// AddRepo parses the .SRCINFO of each dir and adds its packages to the index. +// The first entry to claim a given name wins, so earlier repos mask later ones. +func (i *Index) AddRepo(repoName string, dirs []string) error { + for _, dir := range dirs { + si, err := gosrc.ParseFile(filepath.Join(dir, ".SRCINFO")) + if err != nil { + return fmt.Errorf("parsing %s: %w", filepath.Join(dir, ".SRCINFO"), err) + } + + entry := &Entry{ + RepoName: repoName, + Dir: dir, + Pkgbase: si.Pkgbase, + Version: si.Version(), + Srcinfo: si, + } + + i.claimName(si.Pkgbase, entry) + for _, pkg := range si.SplitPackages() { + i.claimName(pkg.Pkgname, entry) + for _, prov := range pkg.Provides { + i.claimProvide(provideName(prov.Value), entry) + } + } + } + + return nil +} + +// Get resolves a package name to its entry, checking real names before +// provides. +func (i *Index) Get(name string) (*Entry, bool) { + if e, ok := i.byName[name]; ok { + return e, true + } + + if e, ok := i.byProvides[name]; ok { + return e, true + } + + return nil, false +} + +func (i *Index) claimName(name string, entry *Entry) { + if name == "" { + return + } + + if _, exists := i.byName[name]; !exists { + i.byName[name] = entry + } +} + +func (i *Index) claimProvide(name string, entry *Entry) { + if name == "" { + return + } + + if _, exists := i.byProvides[name]; !exists { + i.byProvides[name] = entry + } +} + +// provideName strips any version constraint from a provides value, e.g. +// "libfoo.so=1" or "bar>=2.0" becomes the bare package name. +func provideName(value string) string { + if i := strings.IndexAny(value, "=<>"); i >= 0 { + return value[:i] + } + + return value +} diff --git a/pkg/pkgbuildrepo/index_test.go b/pkg/pkgbuildrepo/index_test.go new file mode 100644 index 000000000..053fba89e --- /dev/null +++ b/pkg/pkgbuildrepo/index_test.go @@ -0,0 +1,80 @@ +//go:build !integration + +package pkgbuildrepo + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func writeSrcinfo(t *testing.T, dir, contents string) { + t.Helper() + require.NoError(t, os.MkdirAll(dir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, ".SRCINFO"), []byte(contents), 0o600)) +} + +// GIVEN repos with a plain package, a split package, and a provides +// WHEN an index is built +// THEN pkgbase, every pkgname, and provides resolve to the owning entry. +func TestIndexMapsNamesBaseAndProvides(t *testing.T) { + t.Parallel() + + root := t.TempDir() + fooDir := filepath.Join(root, "foo") + multiDir := filepath.Join(root, "multi") + + writeSrcinfo(t, fooDir, "pkgbase = foo\n\tpkgver = 1.0\n\tpkgrel = 1\n\tarch = x86_64\n\npkgname = foo\n\tprovides = bar\n") + writeSrcinfo(t, multiDir, "pkgbase = multi\n\tpkgver = 2.0\n\tpkgrel = 3\n\tarch = x86_64\n\npkgname = m1\n\npkgname = m2\n") + + idx := NewIndex() + require.NoError(t, idx.AddRepo("myrepo", []string{fooDir, multiDir})) + + e, ok := idx.Get("foo") + require.True(t, ok) + assert.Equal(t, "foo", e.Pkgbase) + assert.Equal(t, "1.0-1", e.Version) + assert.Equal(t, "myrepo", e.RepoName) + assert.Equal(t, fooDir, e.Dir) + + prov, ok := idx.Get("bar") + require.True(t, ok) + assert.Equal(t, "foo", prov.Pkgbase) + + e1, ok := idx.Get("m1") + require.True(t, ok) + e2, ok := idx.Get("m2") + require.True(t, ok) + assert.Equal(t, "multi", e1.Pkgbase) + assert.Same(t, e1, e2) + + _, ok = idx.Get("multi") + assert.True(t, ok) + + _, ok = idx.Get("does-not-exist") + assert.False(t, ok) +} + +// GIVEN one entry that provides "shared" and another literally named "shared" +// WHEN both are indexed (in either order) +// THEN a lookup of "shared" resolves to the real package, not the provider. +func TestIndexPkgnameBeatsProvides(t *testing.T) { + t.Parallel() + + root := t.TempDir() + providerDir := filepath.Join(root, "provider") + realDir := filepath.Join(root, "shared") + + writeSrcinfo(t, providerDir, "pkgbase = provider\n\tpkgver = 1\n\tpkgrel = 1\n\tarch = x86_64\n\npkgname = provider\n\tprovides = shared=1.0\n") + writeSrcinfo(t, realDir, "pkgbase = shared\n\tpkgver = 1\n\tpkgrel = 1\n\tarch = x86_64\n\npkgname = shared\n") + + idx := NewIndex() + require.NoError(t, idx.AddRepo("r", []string{providerDir, realDir})) + + e, ok := idx.Get("shared") + require.True(t, ok) + assert.Equal(t, "shared", e.Pkgbase) +} diff --git a/pkg/pkgbuildrepo/load.go b/pkg/pkgbuildrepo/load.go new file mode 100644 index 000000000..aeed8880f --- /dev/null +++ b/pkg/pkgbuildrepo/load.go @@ -0,0 +1,45 @@ +package pkgbuildrepo + +import ( + "context" + "fmt" + + "github.com/Jguer/yay/v13/pkg/settings/exe" +) + +// RepoConfig is the subset of a configured PKGBUILD repository needed to +// refresh and scan it. Callers build it from settings.PkgbuildRepo. +type RepoConfig struct { + Name string + URL string + Depth int +} + +// Load refreshes every configured repo and returns an index of the packages +// they provide. Repos are processed in order, so earlier repos mask later ones +// (and all repos mask the AUR downstream). +func Load(ctx context.Context, cmdBuilder exe.ICmdBuilder, + repos []RepoConfig, cacheDir string, refresh bool, +) (*Index, error) { + idx := NewIndex() + + for i := range repos { + repo := repos[i] + + loc, err := Refresh(ctx, cmdBuilder, repo.Name, repo.URL, cacheDir, refresh) + if err != nil { + return nil, fmt.Errorf("refreshing pkgbuild repo %q: %w", repo.Name, err) + } + + dirs, err := Scan(loc.Dir, repo.Depth) + if err != nil { + return nil, fmt.Errorf("scanning pkgbuild repo %q: %w", repo.Name, err) + } + + if err := idx.AddRepo(repo.Name, dirs); err != nil { + return nil, fmt.Errorf("indexing pkgbuild repo %q: %w", repo.Name, err) + } + } + + return idx, nil +} diff --git a/pkg/pkgbuildrepo/load_test.go b/pkg/pkgbuildrepo/load_test.go new file mode 100644 index 000000000..6ccac7db4 --- /dev/null +++ b/pkg/pkgbuildrepo/load_test.go @@ -0,0 +1,67 @@ +//go:build !integration + +package pkgbuildrepo + +import ( + "context" + "os/exec" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/Jguer/yay/v13/pkg/settings/exe" +) + +// GIVEN a local file:// repo containing one package +// WHEN loaded +// THEN the package is scanned and indexed to its directory without any git use. +func TestLoadIndexesLocalRepo(t *testing.T) { + t.Parallel() + + repoDir := t.TempDir() + pkgDir := filepath.Join(repoDir, "foo") + writePkgbuild(t, pkgDir) + writeSrcinfo(t, pkgDir, "pkgbase = foo\n\tpkgver = 1\n\tpkgrel = 1\n\tarch = x86_64\n\npkgname = foo\n") + + runner := &exe.MockRunner{} + builder := &exe.MockBuilder{Runner: runner} + + idx, err := Load(context.Background(), builder, + []RepoConfig{{Name: "myrepo", URL: "file://" + repoDir, Depth: 3}}, + t.TempDir(), false) + require.NoError(t, err) + + e, ok := idx.Get("foo") + require.True(t, ok) + assert.Equal(t, pkgDir, e.Dir) + assert.Equal(t, "myrepo", e.RepoName) + + assert.Empty(t, runner.CaptureCalls) +} + +// GIVEN a repo package that ships a PKGBUILD but no .SRCINFO +// WHEN loaded +// THEN it is rejected without executing the PKGBUILD. +func TestLoadRejectsMissingSrcinfoWithoutExecutingPKGBUILD(t *testing.T) { + t.Parallel() + + repoDir := t.TempDir() + pkgDir := filepath.Join(repoDir, "foo") + writePkgbuild(t, pkgDir) + + runner := &exe.MockRunner{ + CaptureFn: func(*exec.Cmd) (string, string, error) { + t.Fatal("loading must not execute PKGBUILD content") + return "", "", nil + }, + } + builder := &exe.MockBuilder{Runner: runner} + + _, err := Load(context.Background(), builder, + []RepoConfig{{Name: "r", URL: "file://" + repoDir, Depth: 3}}, + t.TempDir(), false) + require.Error(t, err) + assert.Empty(t, runner.CaptureCalls) +} diff --git a/pkg/pkgbuildrepo/location.go b/pkg/pkgbuildrepo/location.go new file mode 100644 index 000000000..807f978ec --- /dev/null +++ b/pkg/pkgbuildrepo/location.go @@ -0,0 +1,94 @@ +package pkgbuildrepo + +import ( + "context" + "os" + "path/filepath" + "strings" + + "github.com/Jguer/yay/v13/pkg/download" + "github.com/Jguer/yay/v13/pkg/settings/exe" +) + +// Location describes where a repository's PKGBUILDs live on disk and how the +// directory is kept up to date. +type Location struct { + // Dir is the local directory scanned for PKGBUILDs. + Dir string + // IsGit reports whether Dir is a git checkout refreshed via clone/pull. + IsGit bool + // CloneURL is the git URL to clone/pull when IsGit is true. + CloneURL string +} + +// Resolve maps a repo's configured URL to a Location, following the makepkg +// source convention: remote schemes and git+file:// point at a git checkout in +// cacheDir/name; plain file:// and bare paths are used in place. +func Resolve(name, url, cacheDir string) Location { + if path, ok := strings.CutPrefix(url, "file://"); ok { + return Location{Dir: path} + } + + if isGitURL(url) { + return Location{ + Dir: filepath.Join(cacheDir, name), + IsGit: true, + CloneURL: strings.TrimPrefix(url, "git+"), + } + } + + return Location{Dir: url} +} + +// Refresh resolves the repo location and, for git repos, ensures the clone +// exists in cacheDir. It always clones a missing repo; an existing clone is +// pulled only when refresh is true, so routine installs work offline while +// -Sy keeps repos up to date. Local directory repos are used in place. +func Refresh(ctx context.Context, cmdBuilder exe.GitCmdBuilder, + name, url, cacheDir string, refresh bool, +) (Location, error) { + loc := Resolve(name, url, cacheDir) + if !loc.IsGit { + return loc, nil + } + + if !refresh { + if _, err := os.Stat(filepath.Join(loc.Dir, ".git")); err == nil { + return loc, nil + } + } + + // git clones into cacheDir/name, so the cache dir must exist first. + if err := os.MkdirAll(cacheDir, 0o755); err != nil { + return loc, err + } + + if _, err := download.PkgbuildRepoClone(ctx, cmdBuilder, loc.CloneURL, name, cacheDir, false); err != nil { + return loc, err + } + + return loc, nil +} + +func isGitURL(url string) bool { + if strings.HasPrefix(url, "git+") { + url = strings.TrimPrefix(url, "git+") + } + + if strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "git://") { + return false + } + if strings.HasPrefix(url, "file://") { + return true + } + + if strings.HasPrefix(url, "https://") || strings.HasPrefix(url, "ssh://") { + return true + } + + if strings.Contains(url, "://") { + return false + } + + return strings.HasSuffix(url, ".git") +} diff --git a/pkg/pkgbuildrepo/location_test.go b/pkg/pkgbuildrepo/location_test.go new file mode 100644 index 000000000..d28dc6d9b --- /dev/null +++ b/pkg/pkgbuildrepo/location_test.go @@ -0,0 +1,68 @@ +//go:build !integration + +package pkgbuildrepo + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +// GIVEN repo URLs in various makepkg-source forms +// WHEN resolved against a cache dir +// THEN git URLs clone into the cache while file:// and bare paths are used in +// place. +func TestResolveLocation(t *testing.T) { + t.Parallel() + + cache := "/cache" + + tests := []struct { + name string + url string + want Location + }{ + { + name: "remote-https", + url: "https://github.com/Jguer/yay-PKGBUILD", + want: Location{Dir: filepath.Join(cache, "remote-https"), IsGit: true, CloneURL: "https://github.com/Jguer/yay-PKGBUILD"}, + }, + { + name: "ssh-dot-git", + url: "ssh://git@example.invalid/repo.git", + want: Location{Dir: filepath.Join(cache, "ssh-dot-git"), IsGit: true, CloneURL: "ssh://git@example.invalid/repo.git"}, + }, + { + name: "local-git", + url: "git+file:///srv/pkgbuild-repo", + want: Location{Dir: filepath.Join(cache, "local-git"), IsGit: true, CloneURL: "file:///srv/pkgbuild-repo"}, + }, + { + name: "local-dir", + url: "file:///srv/pkgbuilds", + want: Location{Dir: "/srv/pkgbuilds", IsGit: false}, + }, + { + name: "bare-path", + url: "/srv/local", + want: Location{Dir: "/srv/local", IsGit: false}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + assert.Equal(t, tt.want, Resolve(tt.name, tt.url, cache)) + }) + } +} + +func TestResolveDoesNotTreatInsecureGitURLsAsRemoteRepositories(t *testing.T) { + t.Parallel() + + cache := "/cache" + for _, url := range []string{"http://example.invalid/repo.git", "git://example.invalid/repo.git", "git+http://example.invalid/repo.git"} { + assert.False(t, Resolve("repo", url, cache).IsGit, url) + } +} diff --git a/pkg/pkgbuildrepo/refresh_test.go b/pkg/pkgbuildrepo/refresh_test.go new file mode 100644 index 000000000..3c201c3a5 --- /dev/null +++ b/pkg/pkgbuildrepo/refresh_test.go @@ -0,0 +1,61 @@ +//go:build !integration + +package pkgbuildrepo + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/Jguer/yay/v13/pkg/settings/exe" +) + +// GIVEN a git-URL repo +// WHEN refreshed into an empty cache +// THEN it is cloned and the scan directory points at the clone. +func TestRefreshClonesGitRepo(t *testing.T) { + t.Parallel() + + // A not-yet-existing cache dir must be created before cloning. + cache := filepath.Join(t.TempDir(), "repos") + runner := &exe.MockRunner{} + builder := &exe.MockBuilder{Runner: runner} + + loc, err := Refresh(context.Background(), builder, "yay-pkgbuild", + "https://github.com/Jguer/yay-PKGBUILD", cache, false) + require.NoError(t, err) + + _, statErr := os.Stat(cache) + require.NoError(t, statErr) + + assert.True(t, loc.IsGit) + assert.Equal(t, filepath.Join(cache, "yay-pkgbuild"), loc.Dir) + + require.Len(t, runner.CaptureCalls, 1) + cmd := runner.CaptureCalls[0].Args[0].(*exec.Cmd) + assert.Contains(t, cmd.Args, "clone") + assert.Contains(t, cmd.Args, "https://github.com/Jguer/yay-PKGBUILD") + assert.Contains(t, cmd.Args, "yay-pkgbuild") +} + +// GIVEN a file:// directory repo +// WHEN refreshed +// THEN no git command runs and the scan directory is the local path. +func TestRefreshLocalDirNoGit(t *testing.T) { + t.Parallel() + + runner := &exe.MockRunner{} + builder := &exe.MockBuilder{Runner: runner} + + loc, err := Refresh(context.Background(), builder, "local", "file:///srv/pkgbuilds", t.TempDir(), false) + require.NoError(t, err) + + assert.False(t, loc.IsGit) + assert.Equal(t, "/srv/pkgbuilds", loc.Dir) + assert.Empty(t, runner.CaptureCalls) +} diff --git a/pkg/pkgbuildrepo/scan.go b/pkg/pkgbuildrepo/scan.go new file mode 100644 index 000000000..bc1369dee --- /dev/null +++ b/pkg/pkgbuildrepo/scan.go @@ -0,0 +1,69 @@ +// Package pkgbuildrepo scans user-configured PKGBUILD repositories and indexes +// the packages they provide so they can mask AUR packages during resolution. +package pkgbuildrepo + +import ( + "io/fs" + "os" + "path/filepath" + "slices" + "strings" +) + +// Scan walks root and returns every directory containing a PKGBUILD, up to +// depth directory levels below root (root itself is level 0). Hidden +// directories such as .git are skipped. The result is sorted for determinism. +func Scan(root string, depth int) ([]string, error) { + var dirs []string + + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + + if !d.IsDir() { + return nil + } + + level := dirLevel(root, path) + + // Skip hidden directories (e.g. .git) and everything beneath them, + // but never skip root even if its name begins with a dot. + if level > 0 && strings.HasPrefix(d.Name(), ".") { + return filepath.SkipDir + } + + if level > depth { + return filepath.SkipDir + } + + if hasPkgbuild(path) { + dirs = append(dirs, path) + } + + return nil + }) + if err != nil { + return nil, err + } + + slices.Sort(dirs) + + return dirs, nil +} + +// dirLevel returns how many directory levels path is below root. +func dirLevel(root, path string) int { + rel, err := filepath.Rel(root, path) + if err != nil || rel == "." { + return 0 + } + + return strings.Count(rel, string(os.PathSeparator)) + 1 +} + +func hasPkgbuild(dir string) bool { + info, err := os.Lstat(filepath.Join(dir, "PKGBUILD")) + + return err == nil && !info.IsDir() && info.Mode()&os.ModeSymlink == 0 +} diff --git a/pkg/pkgbuildrepo/scan_test.go b/pkg/pkgbuildrepo/scan_test.go new file mode 100644 index 000000000..e46291442 --- /dev/null +++ b/pkg/pkgbuildrepo/scan_test.go @@ -0,0 +1,76 @@ +//go:build !integration + +package pkgbuildrepo + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func writePkgbuild(t *testing.T, dir string) { + t.Helper() + require.NoError(t, os.MkdirAll(dir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "PKGBUILD"), []byte("# pkgbuild\n"), 0o600)) +} + +// GIVEN a repo tree with PKGBUILDs at several depths +// WHEN scanned with a depth bound +// THEN only PKGBUILD directories within the bound are returned, sorted, and +// version-control/hidden directories are skipped. +func TestScanRespectsDepthAndSkipsHidden(t *testing.T) { + t.Parallel() + + root := t.TempDir() + writePkgbuild(t, root) // level 0 + writePkgbuild(t, filepath.Join(root, "foo")) // level 1 + writePkgbuild(t, filepath.Join(root, "bar", "baz")) // level 2 + writePkgbuild(t, filepath.Join(root, "a", "b", "c", "d")) // level 4 - excluded + + // A .git dir containing a stray PKGBUILD must not be indexed. + writePkgbuild(t, filepath.Join(root, ".git", "hooksdir")) + + dirs, err := Scan(root, 3) + require.NoError(t, err) + + assert.Equal(t, []string{ + root, + filepath.Join(root, "bar", "baz"), + filepath.Join(root, "foo"), + }, dirs) +} + +// GIVEN a repo with a single top-level PKGBUILD +// WHEN scanned with depth 0 +// THEN only the root is returned. +func TestScanDepthZeroOnlyRoot(t *testing.T) { + t.Parallel() + + root := t.TempDir() + writePkgbuild(t, root) + writePkgbuild(t, filepath.Join(root, "foo")) + + dirs, err := Scan(root, 0) + require.NoError(t, err) + + assert.Equal(t, []string{root}, dirs) +} + +func TestScanIgnoresSymlinkedPKGBUILD(t *testing.T) { + t.Parallel() + + root := t.TempDir() + target := filepath.Join(root, "target") + writePkgbuild(t, target) + + linkDir := filepath.Join(root, "link") + require.NoError(t, os.MkdirAll(linkDir, 0o755)) + require.NoError(t, os.Symlink(filepath.Join(target, "PKGBUILD"), filepath.Join(linkDir, "PKGBUILD"))) + + dirs, err := Scan(root, 1) + require.NoError(t, err) + assert.Equal(t, []string{target}, dirs) +} diff --git a/pkg/settings/config.go b/pkg/settings/config.go index 54946c6e5..68813485d 100644 --- a/pkg/settings/config.go +++ b/pkg/settings/config.go @@ -71,6 +71,10 @@ type Configuration struct { UseRPC bool `json:"rpc" lua:"rpc"` DoubleConfirm bool `json:"doubleconfirm" lua:"double_confirm"` // confirm install before and after build + // PkgbuildRepos is configured only via init.lua (yay.opt.pkgbuild_repos), + // never persisted to config.json. + PkgbuildRepos []PkgbuildRepo `json:"-" lua:"pkgbuild_repos"` + CompletionPath string `json:"-" lua:"-"` VCSFilePath string `json:"-" lua:"-"` // ConfigPath string `json:"-"` diff --git a/pkg/settings/lua/lua.go b/pkg/settings/lua/lua.go index de63c7c29..040d4507a 100644 --- a/pkg/settings/lua/lua.go +++ b/pkg/settings/lua/lua.go @@ -5,6 +5,7 @@ import ( "fmt" "path/filepath" "reflect" + "slices" "strings" "github.com/Jguer/yay/v13/pkg/text" @@ -61,16 +62,7 @@ func (e *Engine) Apply(cfg any) (unknown []string, errs []error) { } sv := v.Elem() - st := sv.Type() - - index := make(map[string]int, st.NumField()) - - for i := range st.NumField() { - field := st.Field(i) - if name := luaKeyForField(&field); name != "" { - index[name] = i - } - } + index := luaFieldIndex(sv.Type()) optTbl, ok := e.optTable() if !ok { @@ -149,6 +141,20 @@ func luaKeyForField(field *reflect.StructField) string { return "" } +// luaFieldIndex maps each lua-tagged field name of st to its field index. +func luaFieldIndex(st reflect.Type) map[string]int { + index := make(map[string]int, st.NumField()) + + for i := range st.NumField() { + field := st.Field(i) + if name := luaKeyForField(&field); name != "" { + index[name] = i + } + } + + return index +} + func assign(field reflect.Value, val lua.LValue) error { switch field.Kind() { case reflect.String: @@ -172,9 +178,123 @@ func assign(field reflect.Value, val lua.LValue) error { } field.SetInt(int64(n)) + case reflect.Slice: + return assignStructSlice(field, val) default: return fmt.Errorf("unsupported field kind %s", field.Kind()) } return nil } + +// assignStructSlice fills a []Struct field from a Lua table keyed by name, e.g. +// +// { ["core"] = { url = "..." }, ["extra"] = { url = "..." } } +// +// Each entry becomes one struct: the table key populates the element's +// lua:"name" field and the sub-table populates the remaining fields. Entries +// are sorted by name so the resulting slice is deterministic despite Lua's +// unordered table iteration. +func assignStructSlice(field reflect.Value, val lua.LValue) error { + elemType := field.Type().Elem() + if elemType.Kind() != reflect.Struct { + return fmt.Errorf("unsupported slice element kind %s", elemType.Kind()) + } + + tbl, ok := val.(*lua.LTable) + if !ok { + return fmt.Errorf("expected table, got %s", val.Type()) + } + + elemIndex := luaFieldIndex(elemType) + + type namedElem struct { + name string + elem reflect.Value + } + + var ( + entries []namedElem + firstErr error + ) + + tbl.ForEach(func(k, entry lua.LValue) { + if firstErr != nil { + return + } + + name, ok := k.(lua.LString) + if !ok { + firstErr = fmt.Errorf("entry keys must be strings, got %s", k.Type()) + return + } + + entryTbl, ok := entry.(*lua.LTable) + if !ok { + firstErr = fmt.Errorf("entry %q must be a table, got %s", string(name), entry.Type()) + return + } + + elem := reflect.New(elemType).Elem() + if nameIdx, found := elemIndex["name"]; found { + elem.Field(nameIdx).SetString(string(name)) + } + + if err := assignStructFields(elem, entryTbl, elemIndex); err != nil { + firstErr = fmt.Errorf("entry %q: %w", string(name), err) + return + } + + entries = append(entries, namedElem{name: string(name), elem: elem}) + }) + + if firstErr != nil { + return firstErr + } + + // Sort by name so the resulting slice is deterministic despite Lua's + // unordered table iteration. + slices.SortFunc(entries, func(a, b namedElem) int { + return strings.Compare(a.name, b.name) + }) + + out := reflect.MakeSlice(field.Type(), len(entries), len(entries)) + for i, entry := range entries { + out.Index(i).Set(entry.elem) + } + + field.Set(out) + + return nil +} + +// assignStructFields assigns the entries of tbl onto struct value sv, matching +// each key against the lua:"..." tags in index. Unknown keys are errors so +// typos in nested option tables fail fast, mirroring top-level opt handling. +func assignStructFields(sv reflect.Value, tbl *lua.LTable, index map[string]int) error { + var firstErr error + + tbl.ForEach(func(k, entry lua.LValue) { + if firstErr != nil { + return + } + + key, ok := k.(lua.LString) + if !ok { + firstErr = fmt.Errorf("keys must be strings, got %s", k.Type()) + return + } + + fieldIdx, found := index[string(key)] + if !found { + firstErr = fmt.Errorf("unknown key %q", string(key)) + return + } + + if err := assign(sv.Field(fieldIdx), entry); err != nil { + firstErr = fmt.Errorf("%s: %w", string(key), err) + } + }) + + return firstErr +} diff --git a/pkg/settings/lua/lua_test.go b/pkg/settings/lua/lua_test.go index a5dd979c5..c37dd42d3 100644 --- a/pkg/settings/lua/lua_test.go +++ b/pkg/settings/lua/lua_test.go @@ -83,6 +83,74 @@ func TestApplyAppliesAnswerOptionsFromLua(t *testing.T) { assert.Equal(t, "Installed", cfg.AnswerEdit) } +type repoTestConfig struct { + BuildDir string `lua:"build_dir"` + PkgbuildRepos []pkgbuildRepo `lua:"pkgbuild_repos"` +} + +type pkgbuildRepo struct { + Name string `lua:"name"` + URL string `lua:"url"` + Depth int `lua:"depth"` +} + +func TestApplyPkgbuildRepos(t *testing.T) { + t.Parallel() + e := New() + t.Cleanup(e.Close) + + require.NoError(t, e.L.DoString(` + yay.opt.build_dir = "/tmp/yay" + yay.opt.pkgbuild_repos = { + ["yay-pkgbuild"] = { + url = "https://github.com/Jguer/yay-PKGBUILD", + depth = 2, + }, + ["local-repo"] = { + url = "file:///srv/pkgbuilds", + }, + } + `)) + + cfg := &repoTestConfig{} + unknown, errs := e.Apply(cfg) + + assert.Empty(t, unknown) + assert.Empty(t, errs) + assert.Equal(t, "/tmp/yay", cfg.BuildDir) + + // The keyed table becomes a slice sorted by repo name for determinism. + require.Len(t, cfg.PkgbuildRepos, 2) + + assert.Equal(t, "local-repo", cfg.PkgbuildRepos[0].Name) + assert.Equal(t, "file:///srv/pkgbuilds", cfg.PkgbuildRepos[0].URL) + assert.Equal(t, 0, cfg.PkgbuildRepos[0].Depth) + + assert.Equal(t, "yay-pkgbuild", cfg.PkgbuildRepos[1].Name) + assert.Equal(t, "https://github.com/Jguer/yay-PKGBUILD", cfg.PkgbuildRepos[1].URL) + assert.Equal(t, 2, cfg.PkgbuildRepos[1].Depth) +} + +func TestApplyPkgbuildReposRejectsUnknownRepoKey(t *testing.T) { + t.Parallel() + e := New() + t.Cleanup(e.Close) + + require.NoError(t, e.L.DoString(` + yay.opt.pkgbuild_repos = { + ["yay-pkgbuild"] = { + url = "https://github.com/Jguer/yay-PKGBUILD", + nonsense = true, + }, + } + `)) + + cfg := &repoTestConfig{} + _, errs := e.Apply(cfg) + + assert.Len(t, errs, 1) +} + func TestApplyRejectsNonPointer(t *testing.T) { t.Parallel() e := New() diff --git a/pkg/settings/pkgbuild_repo.go b/pkg/settings/pkgbuild_repo.go new file mode 100644 index 000000000..9b7062a78 --- /dev/null +++ b/pkg/settings/pkgbuild_repo.go @@ -0,0 +1,42 @@ +package settings + +import ( + "fmt" + "path/filepath" + "strings" +) + +// DefaultPkgbuildRepoDepth is the recursive PKGBUILD scan depth used when a +// repo does not set one explicitly. +const DefaultPkgbuildRepoDepth = 3 + +// PkgbuildRepo is a user-configured PKGBUILD repository. Packages found in a +// PKGBUILD repository take priority over the AUR, so a repo can mask an AUR +// package. Repositories are configured only through init.lua via +// yay.opt.pkgbuild_repos. +type PkgbuildRepo struct { + // Name identifies the repo; it comes from the pkgbuild_repos table key. + Name string `json:"name" lua:"name"` + // URL locates the repo, following the makepkg source convention: an https + // git URL, a git+file:// local git repo, or a file:// local directory used + // in place. + URL string `json:"url" lua:"url"` + // Depth is how many directory levels deep yay scans for PKGBUILDs. + Depth int `json:"depth" lua:"depth"` +} + +// NormalizePkgbuildRepos validates repository names and fills in default depths. +func (c *Configuration) NormalizePkgbuildRepos() error { + for i := range c.PkgbuildRepos { + name := c.PkgbuildRepos[i].Name + if name == "" || strings.HasPrefix(name, "-") || !filepath.IsLocal(name) || filepath.Base(name) != name { + return fmt.Errorf("invalid PKGBUILD repository name %q", name) + } + + if c.PkgbuildRepos[i].Depth <= 0 { + c.PkgbuildRepos[i].Depth = DefaultPkgbuildRepoDepth + } + } + + return nil +} diff --git a/pkg/settings/pkgbuild_repo_test.go b/pkg/settings/pkgbuild_repo_test.go new file mode 100644 index 000000000..5e2033788 --- /dev/null +++ b/pkg/settings/pkgbuild_repo_test.go @@ -0,0 +1,69 @@ +//go:build !integration + +package settings + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/Jguer/yay/v13/pkg/settings/lua" +) + +// GIVEN an init.lua declaring pkgbuild_repos +// WHEN it is loaded onto a Configuration +// THEN the repos are applied, keyed by name and sorted deterministically +func TestPkgbuildReposFromLua(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + luaPath := filepath.Join(dir, "init.lua") + require.NoError(t, os.WriteFile(luaPath, []byte(` + yay.opt.pkgbuild_repos = { + ["yay-pkgbuild"] = { url = "https://github.com/Jguer/yay-PKGBUILD", depth = 2 }, + ["local-repo"] = { url = "file:///srv/pkgbuilds" }, + } + `), 0o600)) + + cfg := DefaultConfig("test") + require.NoError(t, lua.LoadInto(nil, luaPath, cfg)) + + require.Len(t, cfg.PkgbuildRepos, 2) + + assert.Equal(t, "local-repo", cfg.PkgbuildRepos[0].Name) + assert.Equal(t, "file:///srv/pkgbuilds", cfg.PkgbuildRepos[0].URL) + + assert.Equal(t, "yay-pkgbuild", cfg.PkgbuildRepos[1].Name) + assert.Equal(t, "https://github.com/Jguer/yay-PKGBUILD", cfg.PkgbuildRepos[1].URL) + assert.Equal(t, 2, cfg.PkgbuildRepos[1].Depth) +} + +// GIVEN repos where some omit depth +// WHEN NormalizePkgbuildRepos runs +// THEN missing depths fall back to the default of 3 and explicit depths are kept +func TestNormalizePkgbuildReposDefaultsDepth(t *testing.T) { + t.Parallel() + + cfg := DefaultConfig("test") + cfg.PkgbuildRepos = []PkgbuildRepo{ + {Name: "a", URL: "https://example.invalid/a"}, + {Name: "b", URL: "https://example.invalid/b", Depth: 1}, + } + + require.NoError(t, cfg.NormalizePkgbuildRepos()) + + assert.Equal(t, DefaultPkgbuildRepoDepth, cfg.PkgbuildRepos[0].Depth) + assert.Equal(t, 1, cfg.PkgbuildRepos[1].Depth) +} + +func TestNormalizePkgbuildReposRejectsUnsafeName(t *testing.T) { + t.Parallel() + + cfg := DefaultConfig("test") + cfg.PkgbuildRepos = []PkgbuildRepo{{Name: "../outside", URL: "https://example.invalid/a"}} + + assert.Error(t, cfg.NormalizePkgbuildRepos()) +} diff --git a/pkg/sync/build/installer.go b/pkg/sync/build/installer.go index 3bd1a59bc..4e17d84b1 100644 --- a/pkg/sync/build/installer.go +++ b/pkg/sync/build/installer.go @@ -160,7 +160,7 @@ func (installer *Installer) handleLayer(ctx context.Context, upgradeSync := false for name, info := range layer { switch info.Source { - case dep.AUR, dep.SrcInfo: + case dep.AUR, dep.SrcInfo, dep.PkgbuildRepo: nameToBaseMap[name] = info.AURBase if installer.origTargets.Contains(name) { aurOrigTargetBases.Add(info.AURBase) diff --git a/pkg/sync/post_install.go b/pkg/sync/post_install.go index f0db8d8d9..81c1f6251 100644 --- a/pkg/sync/post_install.go +++ b/pkg/sync/post_install.go @@ -42,6 +42,8 @@ func luaSource(source dep.Source) string { return "local" case dep.SrcInfo: return "srcinfo" + case dep.PkgbuildRepo: + return "pkgbuild_repo" default: return "missing" } diff --git a/pkg/sync/sync.go b/pkg/sync/sync.go index 33ce18614..64619962b 100644 --- a/pkg/sync/sync.go +++ b/pkg/sync/sync.go @@ -70,7 +70,7 @@ func (o *OperationService) Run(ctx context.Context, run *runtime.Runtime, installer.AddPostInstallHook(cleanFunc) } - if cleanAURDirsFunc := preparer.ShouldCleanAURDirs(run, pkgBuildDirs); cleanAURDirsFunc != nil { + if cleanAURDirsFunc := preparer.ShouldCleanAURDirs(run, pkgBuildDirs, targets); cleanAURDirsFunc != nil { installer.AddPostInstallHook(cleanAURDirsFunc) } diff --git a/pkg/sync/workdir/aur_preinstall.go b/pkg/sync/workdir/aur_preinstall.go index 7528dc2b0..89c5256d7 100644 --- a/pkg/sync/workdir/aur_preinstall.go +++ b/pkg/sync/workdir/aur_preinstall.go @@ -152,7 +152,7 @@ func aurTargetPackagesByBase(targets []map[string]*dep.InstallInfo) map[string][ continue } - if info.Source != dep.AUR && info.Source != dep.SrcInfo { + if !info.Source.IsBuiltFromSource() { continue } diff --git a/pkg/sync/workdir/preparer.go b/pkg/sync/workdir/preparer.go index c8c204766..c6d5e41c7 100644 --- a/pkg/sync/workdir/preparer.go +++ b/pkg/sync/workdir/preparer.go @@ -97,15 +97,30 @@ func NewPreparer(dbExecutor db.Executor, cmdBuilder exe.ICmdBuilder, return preper } -func (preper *Preparer) ShouldCleanAURDirs(run *runtime.Runtime, pkgBuildDirs map[string]string) build.PostInstallHookFunc { - if !preper.cfg.CleanAfter || len(pkgBuildDirs) == 0 { +func (preper *Preparer) ShouldCleanAURDirs(run *runtime.Runtime, pkgBuildDirs map[string]string, + targets []map[string]*dep.InstallInfo, +) build.PostInstallHookFunc { + cleanDirs := make(map[string]string, len(pkgBuildDirs)) + for _, layer := range targets { + for _, info := range layer { + if info.Source != dep.AUR && info.Source != dep.SrcInfo { + continue + } + + if dir, ok := pkgBuildDirs[info.AURBase]; ok { + cleanDirs[info.AURBase] = dir + } + } + } + + if !preper.cfg.CleanAfter || len(cleanDirs) == 0 { return nil } - preper.log.Debugln("added post install hook to clean up AUR dirs", pkgBuildDirs) + preper.log.Debugln("added post install hook to clean up AUR dirs", cleanDirs) return func(ctx context.Context) error { - cleanAfter(ctx, run, run.CmdBuilder, pkgBuildDirs) + cleanAfter(ctx, run, run.CmdBuilder, cleanDirs) return nil } } @@ -191,6 +206,11 @@ func (preper *Preparer) PrepareWorkspace(ctx context.Context, ) (map[string]string, error) { aurBasesToClone := mapset.NewThreadUnsafeSet[string]() pkgBuildDirsByBase := make(map[string]string, len(targets)) + // mergeDirsByBase holds the per-package git clones (AUR and local SrcInfo + // checkouts) that get a git reset/merge. PKGBUILD-repo directories are + // managed by the repo refresh — they are either a plain directory or a + // subdirectory of a shared clone — so they must not be merged per package. + mergeDirsByBase := make(map[string]string, len(targets)) for _, layer := range targets { for _, info := range layer { @@ -202,9 +222,14 @@ func (preper *Preparer) PrepareWorkspace(ctx context.Context, aurBasesToClone.Add(pkgBase) } pkgBuildDirsByBase[pkgBase] = pkgBuildDir + mergeDirsByBase[pkgBase] = pkgBuildDir case dep.SrcInfo: pkgBase := info.AURBase pkgBuildDirsByBase[pkgBase] = info.SrcinfoPath + mergeDirsByBase[pkgBase] = info.SrcinfoPath + case dep.PkgbuildRepo: + pkgBase := info.AURBase + pkgBuildDirsByBase[pkgBase] = info.SrcinfoPath } } } @@ -219,7 +244,7 @@ func (preper *Preparer) PrepareWorkspace(ctx context.Context, return pkgBuildDirsByBase, nil } - if err := mergePkgbuilds(ctx, preper.cmdBuilder, pkgBuildDirsByBase); err != nil { + if err := mergePkgbuilds(ctx, preper.cmdBuilder, mergeDirsByBase); err != nil { return nil, err } diff --git a/pkg/sync/workdir/preparer_pkgbuild_repo_test.go b/pkg/sync/workdir/preparer_pkgbuild_repo_test.go new file mode 100644 index 000000000..a7f782acb --- /dev/null +++ b/pkg/sync/workdir/preparer_pkgbuild_repo_test.go @@ -0,0 +1,138 @@ +//go:build !integration + +package workdir + +import ( + "os/exec" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + glua "github.com/yuin/gopher-lua" + + "github.com/Jguer/yay/v13/pkg/db/mock" + "github.com/Jguer/yay/v13/pkg/dep" + "github.com/Jguer/yay/v13/pkg/runtime" + "github.com/Jguer/yay/v13/pkg/settings" + "github.com/Jguer/yay/v13/pkg/settings/exe" + settingslua "github.com/Jguer/yay/v13/pkg/settings/lua" +) + +// GIVEN a PKGBUILD-repo target whose PKGBUILD is already local +// WHEN the workspace is prepared +// THEN its build dir is the repo directory and no git clone is performed. +func TestPrepareWorkspacePkgbuildRepoUsesLocalDir(t *testing.T) { + t.Parallel() + + repoDir := t.TempDir() + cfg := &settings.Configuration{BuildDir: t.TempDir(), ReDownload: "no"} + + runner := &exe.MockRunner{} + builder := &exe.MockBuilder{Runner: runner} + + preper := NewPreparerWithoutHooks(&mock.DBExecutor{}, builder, cfg, newTestLogger(), false) + + targets := []map[string]*dep.InstallInfo{ + {"foo": {Source: dep.PkgbuildRepo, AURBase: "foo", SrcinfoPath: repoDir, Version: "1-1"}}, + } + + dirs, err := preper.PrepareWorkspace(t.Context(), nil, targets) + require.NoError(t, err) + + assert.Equal(t, repoDir, dirs["foo"]) + assert.Empty(t, runner.CaptureCalls) +} + +// GIVEN a PKGBUILD-repo target and source downloading enabled +// WHEN the workspace is prepared +// THEN the repo directory is never git reset/merged (it is not a per-package +// clone), unlike AUR/SrcInfo checkouts. +func TestPrepareWorkspacePkgbuildRepoNotMerged(t *testing.T) { + t.Parallel() + + repoDir := t.TempDir() + cfg := &settings.Configuration{BuildDir: t.TempDir(), ReDownload: "no", MaxConcurrentDownloads: 1} + + runner := &exe.MockRunner{} + builder := &exe.MockBuilder{Runner: runner} + + dbExe := &mock.DBExecutor{InstalledRemotePackageNamesFn: func() []string { return nil }} + preper := NewPreparerWithoutHooks(dbExe, builder, cfg, newTestLogger(), true) + + run := &runtime.Runtime{Cfg: cfg, Logger: newTestLogger(), CmdBuilder: builder} + + targets := []map[string]*dep.InstallInfo{ + {"foo": {Source: dep.PkgbuildRepo, AURBase: "foo", SrcinfoPath: repoDir, Version: "1-1"}}, + } + + dirs, err := preper.PrepareWorkspace(t.Context(), run, targets) + require.NoError(t, err) + assert.Equal(t, repoDir, dirs["foo"]) + + for _, call := range runner.CaptureCalls { + cmd := call.Args[0].(*exec.Cmd) + assert.NotContains(t, cmd.Args, "reset", "repo dir must not be git reset") + assert.NotContains(t, cmd.Args, "merge", "repo dir must not be git merged") + } +} + +// GIVEN a PKGBUILD-repo target and Lua autocmds registered for AURPreInstall +// and AURPostDownload +// WHEN the workspace is prepared +// THEN both hooks fire for the pkgbuild-repo base, proving Lua hook +// invocation is not gated to dep.AUR/dep.SrcInfo sources. +func TestPrepareWorkspacePkgbuildRepoFiresLuaHooks(t *testing.T) { + t.Parallel() + + base := "foo" + repoDir := writeAURPreInstallPackage(t, base) + cfg := &settings.Configuration{BuildDir: t.TempDir(), ReDownload: "no", MaxConcurrentDownloads: 1} + + runner := &exe.MockRunner{} + builder := &exe.MockBuilder{Runner: runner} + + dbExe := &mock.DBExecutor{InstalledRemotePackageNamesFn: func() []string { return nil }} + preper := NewPreparerWithoutHooks(dbExe, builder, cfg, newTestLogger(), true) + + engine := settingslua.New() + t.Cleanup(engine.Close) + + fired := []string{} + engine.L.SetGlobal("record", engine.L.NewFunction(func(L *glua.LState) int { + fired = append(fired, L.CheckString(1)) + return 0 + })) + require.NoError(t, engine.L.DoString(` + yay.create_autocmd("AURPreInstall", { + callback = function(event) record("pre:" .. event.match) end, + }) + yay.create_autocmd("AURPostDownload", { + callback = function(event) record("post:" .. event.match) end, + }) + `)) + + run := &runtime.Runtime{Cfg: cfg, Logger: newTestLogger(), CmdBuilder: builder, Lua: engine} + + targets := []map[string]*dep.InstallInfo{ + {"demo": {Source: dep.PkgbuildRepo, AURBase: base, SrcinfoPath: repoDir, Version: "1:1.2.3-4"}}, + } + + dirs, err := preper.PrepareWorkspace(t.Context(), run, targets) + require.NoError(t, err) + assert.Equal(t, repoDir, dirs[base]) + assert.Equal(t, []string{"pre:" + base, "post:" + base}, fired) +} + +func TestShouldCleanAURDirsExcludesPkgbuildRepos(t *testing.T) { + t.Parallel() + + repoDir := t.TempDir() + preper := NewPreparerWithoutHooks(&mock.DBExecutor{}, &exe.MockBuilder{}, + &settings.Configuration{CleanAfter: true}, newTestLogger(), false) + + hook := preper.ShouldCleanAURDirs(nil, map[string]string{"foo": repoDir}, []map[string]*dep.InstallInfo{ + {"foo": {Source: dep.PkgbuildRepo, AURBase: "foo", SrcinfoPath: repoDir}}, + }) + + assert.Nil(t, hook) +} diff --git a/pkg/upgrade/pkgbuild_repo_test.go b/pkg/upgrade/pkgbuild_repo_test.go new file mode 100644 index 000000000..6a7211259 --- /dev/null +++ b/pkg/upgrade/pkgbuild_repo_test.go @@ -0,0 +1,141 @@ +//go:build !integration + +package upgrade + +import ( + "context" + "io" + "os" + "path/filepath" + "strings" + "testing" + + aur "github.com/Jguer/aur" + alpm "github.com/Jguer/dyalpm" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/Jguer/yay/v13/pkg/db" + "github.com/Jguer/yay/v13/pkg/db/mock" + "github.com/Jguer/yay/v13/pkg/dep" + mockaur "github.com/Jguer/yay/v13/pkg/dep/mock" + "github.com/Jguer/yay/v13/pkg/pkgbuildrepo" + "github.com/Jguer/yay/v13/pkg/query" + "github.com/Jguer/yay/v13/pkg/settings" + "github.com/Jguer/yay/v13/pkg/settings/parser" + "github.com/Jguer/yay/v13/pkg/text" + "github.com/Jguer/yay/v13/pkg/vcs" +) + +// GIVEN an installed package owned by a PKGBUILD repo with a newer version +// WHEN upgrades are graphed +// THEN it is queued as a PkgbuildRepo upgrade and the AUR is never asked about it. +func TestUpgradeService_pkgbuildRepoUpgradeMasksAUR(t *testing.T) { + t.Parallel() + + pkgDir := filepath.Join(t.TempDir(), "foo") + require.NoError(t, os.MkdirAll(pkgDir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(pkgDir, ".SRCINFO"), + []byte("pkgbase = foo\n\tpkgver = 2\n\tpkgrel = 1\n\tarch = x86_64\n\npkgname = foo\n"), 0o600)) + + idx := pkgbuildrepo.NewIndex() + require.NoError(t, idx.AddRepo("myrepo", []string{pkgDir})) + + dbExe := &mock.DBExecutor{ + AlpmArchitecturesFn: func() ([]string, error) { return []string{"x86_64"}, nil }, + InstalledRemotePackageNamesFn: func() []string { return []string{"foo"} }, + InstalledRemotePackagesFn: func() map[string]mock.IPackage { + return map[string]mock.IPackage{ + "foo": &mock.Package{PName: "foo", PBase: "foo", PVersion: "1-1", PReason: alpm.PkgReasonExplicit}, + } + }, + LocalSatisfierExistsFn: func(string) bool { return false }, + SyncUpgradesFn: func(bool) (map[string]db.SyncUpgrade, error) { return map[string]db.SyncUpgrade{}, nil }, + ReposFn: func() []string { return nil }, + } + + mockAUR := &mockaur.MockAUR{GetFn: func(_ context.Context, q *aur.Query) ([]aur.Pkg, error) { + for _, needle := range q.Needles { + if needle == "foo" { + t.Errorf("AUR must not be queried for repo-owned package foo, got %+v", q) + } + } + return []aur.Pkg{}, nil + }} + + logger := text.NewLogger(io.Discard, os.Stderr, strings.NewReader(""), false, "test") + grapher := dep.NewGrapher(dbExe, mockAUR, false, true, false, false, false, logger) + grapher.SetPkgbuildRepos(idx) + + u := &UpgradeService{ + log: logger, + grapher: grapher, + aurCache: mockAUR, + dbExecutor: dbExe, + vcsStore: &vcs.Mock{}, + cfg: &settings.Configuration{Mode: parser.ModeAny}, + AURWarnings: query.NewWarnings(logger), + } + + graph, err := u.GraphUpgrades(t.Context(), nil, false, func(*Upgrade) bool { return true }) + require.NoError(t, err) + + info := graph.GetNodeInfo("foo") + require.NotNil(t, info) + require.NotNil(t, info.Value) + assert.Equal(t, dep.PkgbuildRepo, info.Value.Source) + assert.True(t, info.Value.Upgrade) + assert.Equal(t, "1-1", info.Value.LocalVersion) + assert.Equal(t, "2-1", info.Value.Version) + + // Repo-only mode must not build packages from PKGBUILD repositories. + u.cfg.Mode = parser.ModeRepo + graph, err = u.GraphUpgrades(t.Context(), nil, false, func(*Upgrade) bool { return true }) + require.NoError(t, err) + assert.Nil(t, graph.GetNodeInfo("foo")) +} + +func TestUpgradeService_pkgbuildRepoDowngrade(t *testing.T) { + t.Parallel() + + pkgDir := filepath.Join(t.TempDir(), "foo") + require.NoError(t, os.MkdirAll(pkgDir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(pkgDir, ".SRCINFO"), + []byte("pkgbase = foo\n\tpkgver = 2\n\tpkgrel = 1\n\tarch = x86_64\n\npkgname = foo\n"), 0o600)) + idx := pkgbuildrepo.NewIndex() + require.NoError(t, idx.AddRepo("myrepo", []string{pkgDir})) + + logger := text.NewLogger(io.Discard, os.Stderr, strings.NewReader(""), false, "test") + grapher := dep.NewGrapher(&mock.DBExecutor{ + AlpmArchitecturesFn: func() ([]string, error) { return []string{"x86_64"}, nil }, + LocalSatisfierExistsFn: func(string) bool { return false }, + }, &mockaur.MockAUR{}, false, true, false, false, false, logger) + grapher.SetPkgbuildRepos(idx) + u := &UpgradeService{grapher: grapher} + + remote := map[string]db.IPackage{ + "foo": &mock.Package{PName: "foo", PBase: "foo", PVersion: "3-1", PReason: alpm.PkgReasonExplicit}, + } + + var errs []error + graph := dep.NewGraph() + u.graphPkgbuildRepoUpgrades(t.Context(), graph, remote, false, nil, &errs) + assert.Empty(t, errs) + assert.Nil(t, graph.GetNodeInfo("foo")) + + graph = dep.NewGraph() + u.graphPkgbuildRepoUpgrades(t.Context(), graph, remote, true, nil, &errs) + info := graph.GetNodeInfo("foo") + require.NotNil(t, info) + require.NotNil(t, info.Value) + assert.True(t, info.Value.Upgrade) + assert.Equal(t, "3-1", info.Value.LocalVersion) + assert.Equal(t, "2-1", info.Value.Version) + + equalRemote := map[string]db.IPackage{ + "foo": &mock.Package{PName: "foo", PBase: "foo", PVersion: "2-1", PReason: alpm.PkgReasonExplicit}, + } + graph = dep.NewGraph() + u.graphPkgbuildRepoUpgrades(t.Context(), graph, equalRemote, true, nil, &errs) + assert.Nil(t, graph.GetNodeInfo("foo")) +} diff --git a/pkg/upgrade/service.go b/pkg/upgrade/service.go index a93abc813..908cb895f 100644 --- a/pkg/upgrade/service.go +++ b/pkg/upgrade/service.go @@ -79,21 +79,15 @@ func (u *UpgradeService) upGraph(ctx context.Context, graph *topo.Graph[string, errs = append(errs, err) replaced := u.syncReplacedPackageNames(syncUpgrades) - if replaced.Cardinality() > 0 { - filteredRemote := make(map[string]db.IPackage, len(remote)) - for name, pkg := range remote { - if !replaced.Contains(name) { - filteredRemote[name] = pkg - } - } - remote = filteredRemote - remoteNames = slices.DeleteFunc(slices.Clone(remoteNames), func(name string) bool { - return replaced.Contains(name) - }) - } + remote, remoteNames = dropFromRemote(remote, remoteNames, replaced) } if u.cfg.Mode.AtLeastAUR() { + // PKGBUILD repositories mask the AUR: handle installed packages owned by a + // configured repo before querying the AUR so they are never double-counted. + repoMasked := u.graphPkgbuildRepoUpgrades(ctx, graph, remote, enableDowngrade, filter, &errs) + remote, remoteNames = dropFromRemote(remote, remoteNames, repoMasked) + u.log.OperationInfoln(gotext.Get("Searching AUR for updates...")) _aurdata, err := u.aurCache.Get(ctx, &aur.Query{Needles: remoteNames, By: aur.Name}) @@ -208,6 +202,81 @@ func (u *UpgradeService) upGraph(ctx context.Context, graph *topo.Graph[string, return errors.Join(errs...) } +// dropFromRemote removes drop's members from the remote package map and name +// slice, returning the filtered copies (or the originals when drop is empty). +func dropFromRemote(remote map[string]db.IPackage, remoteNames []string, + drop mapset.Set[string], +) (filteredRemote map[string]db.IPackage, filteredNames []string) { + if drop.Cardinality() == 0 { + return remote, remoteNames + } + + filtered := make(map[string]db.IPackage, len(remote)) + for name, pkg := range remote { + if !drop.Contains(name) { + filtered[name] = pkg + } + } + + names := slices.DeleteFunc(slices.Clone(remoteNames), func(name string) bool { + return drop.Contains(name) + }) + + return filtered, names +} + +// graphPkgbuildRepoUpgrades graphs upgrades for installed packages owned by a +// configured PKGBUILD repo and returns the set of repo-owned installed package +// names, which the caller strips from the AUR check so repos mask the AUR. +func (u *UpgradeService) graphPkgbuildRepoUpgrades(ctx context.Context, + graph *topo.Graph[string, *dep.InstallInfo], remote map[string]db.IPackage, + enableDowngrade bool, filter Filter, errs *[]error, +) mapset.Set[string] { + masked := mapset.NewThreadUnsafeSet[string]() + if !u.grapher.HasPkgbuildRepos() { + return masked + } + + for name, localPkg := range remote { + entry, ok := u.grapher.PkgbuildRepoEntry(name) + if !ok { + continue + } + + // The repo owns this package; never let the AUR upgrade it. + masked.Add(name) + + localVersion := localPkg.Version() + + cmp := db.VerCmp(entry.Version, localVersion) + if cmp == 0 || (cmp < 0 && !enableDowngrade) { + continue // up to date (or a downgrade the user did not request) + } + + reason := dep.Explicit + if localPkg.Reason() == alpm.PkgReasonDepend { + reason = dep.Dep + } + + if filter != nil && !filter(&db.Upgrade{ + Name: name, + RemoteVersion: entry.Version, + Repository: entry.RepoName, + Base: entry.Pkgbase, + LocalVersion: localVersion, + Reason: localPkg.Reason(), + }) { + continue + } + + if _, err := u.grapher.GraphPkgbuildRepoUpgrade(ctx, graph, entry, name, localVersion, reason); err != nil { + *errs = append(*errs, err) + } + } + + return masked +} + func (u *UpgradeService) syncReplacedPackageNames(syncUpgrades map[string]db.SyncUpgrade) mapset.Set[string] { replaced := mapset.NewThreadUnsafeSet[string]() for _, up := range syncUpgrades { @@ -266,6 +335,16 @@ func (u *UpgradeService) graphToUpSlice(graph *topo.Graph[string, *dep.InstallIn Reason: alpmReason, Extra: extra, }) + case dep.PkgbuildRepo: + aurUp.Up = append(aurUp.Up, Upgrade{ + Name: name, + RemoteVersion: info.Version, + Repository: info.RepoName, + Base: info.AURBase, + LocalVersion: info.LocalVersion, + Reason: alpmReason, + Extra: extra, + }) } return nil }) diff --git a/pkgbuild_repo.go b/pkgbuild_repo.go new file mode 100644 index 000000000..77dcee685 --- /dev/null +++ b/pkgbuild_repo.go @@ -0,0 +1,32 @@ +package main + +import ( + "context" + "path/filepath" + + "github.com/Jguer/yay/v13/pkg/pkgbuildrepo" + "github.com/Jguer/yay/v13/pkg/runtime" +) + +// pkgbuildReposCacheDir is the subdirectory of BuildDir where remote PKGBUILD +// repositories are cloned. +const pkgbuildReposCacheDir = ".pkgbuild-repos" + +// loadPkgbuildRepoIndex builds the index of configured PKGBUILD repositories, +// refreshing git repos when refresh is set. It returns nil when no repos are +// configured. +func loadPkgbuildRepoIndex(ctx context.Context, run *runtime.Runtime, refresh bool) (*pkgbuildrepo.Index, error) { + if len(run.Cfg.PkgbuildRepos) == 0 { + return nil, nil + } + + repos := make([]pkgbuildrepo.RepoConfig, len(run.Cfg.PkgbuildRepos)) + for i := range run.Cfg.PkgbuildRepos { + r := run.Cfg.PkgbuildRepos[i] + repos[i] = pkgbuildrepo.RepoConfig{Name: r.Name, URL: r.URL, Depth: r.Depth} + } + + cacheDir := filepath.Join(run.Cfg.BuildDir, pkgbuildReposCacheDir) + + return pkgbuildrepo.Load(ctx, run.CmdBuilder, repos, cacheDir, refresh) +} diff --git a/print.go b/print.go index 0a8fd9f9b..3fda9f648 100644 --- a/print.go +++ b/print.go @@ -163,7 +163,7 @@ func printUpdateList(ctx context.Context, run *runtime.Runtime, cmdArgs *parser. if noTargets || targets.Contains(pkgName) { if ii.Source == dep.Sync && foreignFilter { return nil - } else if ii.Source == dep.AUR && nativeFilter { + } else if ii.Source.IsBuiltFromSource() && nativeFilter { return nil } diff --git a/sync.go b/sync.go index 5057808a5..9561137cb 100644 --- a/sync.go +++ b/sync.go @@ -46,6 +46,12 @@ func syncInstall(ctx context.Context, grapher := dep.NewGrapher(dbExecutor, aurCache, false, settings.NoConfirm, noDeps, noCheck, cmdArgs.ExistsArg("needed"), run.Logger.Child("grapher")) + pkgbuildRepos, err := loadPkgbuildRepoIndex(ctx, run, refreshArg) + if err != nil { + return fmt.Errorf("%s: %w", gotext.Get("error loading PKGBUILD repositories"), err) + } + grapher.SetPkgbuildRepos(pkgbuildRepos) + graph, err := grapher.GraphFromTargets(ctx, nil, cmdArgs.Targets) if err != nil { return err