Give each site its own VPN exit

.goreleaser.yml

@@ -63,6 +63,12 @@ archives:
       - src: packaging/linux/magnetowid.{service,sysusers,env}
         dst: systemd
         strip_parent: true
+      - src: packaging/linux/magnetowid-vpn@.service
+        dst: systemd
+        strip_parent: true
+      - src: vpn.env.example
+        dst: systemd
+        strip_parent: true
 
 checksum:
   name_template: '{{ .ProjectName }}_{{ .Version }}_SHA256SUMS'
@@ -106,6 +112,13 @@ nfpms:
         dst: /usr/lib/systemd/system/magnetowid.service
       - src: packaging/linux/magnetowid.sysusers
         dst: /usr/lib/sysusers.d/magnetowid.conf
+      - src: packaging/linux/magnetowid-vpn@.service
+        dst: /usr/lib/systemd/system/magnetowid-vpn@.service
+      # Outside /usr/share/doc, which minimal installs leave out.
+      - dst: /usr/share/magnetowid
+        type: dir
+      - src: vpn.env.example
+        dst: /usr/share/magnetowid/vpn.env.example
       - dst: /etc/magnetowid
         type: dir
       - src: packaging/linux/magnetowid.env
@@ -135,14 +148,18 @@ aurs:
     depends:
       - ca-certificates
       - ffmpeg
+    optdepends:
+      - 'docker: VPN exits for region-locked sites'
     backup:
       - etc/magnetowid/magnetowid.env
     install: packaging/linux/magnetowid.install
     package: |-
       install -Dm755 "./magnetowid" "${pkgdir}/usr/bin/magnetowid"
       install -Dm644 "./LICENSE" "${pkgdir}/usr/share/licenses/magnetowid-bin/LICENSE"
       install -Dm644 "./systemd/magnetowid.service" "${pkgdir}/usr/lib/systemd/system/magnetowid.service"
+      install -Dm644 "./systemd/magnetowid-vpn@.service" "${pkgdir}/usr/lib/systemd/system/magnetowid-vpn@.service"
       install -Dm644 "./systemd/magnetowid.sysusers" "${pkgdir}/usr/lib/sysusers.d/magnetowid.conf"
+      install -Dm644 "./systemd/vpn.env.example" "${pkgdir}/usr/share/magnetowid/vpn.env.example"
       install -Dm600 "./systemd/magnetowid.env" "${pkgdir}/etc/magnetowid/magnetowid.env"
 
 homebrew_casks:

README.md

@@ -44,11 +44,12 @@ itself. No separate SABnzbd installation is needed.
 
 | Site | Indexer | Good to know |
 |---|---|---|
-| **TVP VOD** | `/tvp` | Polish series and films. [TVP VOD notes](internal/provider/tvp/README.md) |
+| **TVP VOD** | `/tvp` | Polish series and films. Streams may need a Polish connection. [TVP VOD notes](internal/provider/tvp/README.md) |
 | **BBC iPlayer** | `/bbc` | Streams need a UK connection. [BBC iPlayer notes](internal/provider/bbc/README.md) |
 
 Only what a site offers for free is found: DRM-protected, paid and
-region-blocked streams are left out of the results.
+region-blocked streams are left out of the results. To use a site from another
+country, give it a VPN exit: see [Region-locked sites](docs/vpn.md).
 
 ## What you get
 
@@ -156,6 +157,7 @@ See [Web interface](docs/usage.md#web-interface) for the details.
 ## Documentation
 
 - [Setup and configuration](docs/usage.md): installation options and every setting.
+- [Region-locked sites](docs/vpn.md): a VPN exit for each site's country.
 - [Correcting matches](docs/usage.md#correcting-matches): overrides for titles, seasons and single episodes.
 - [Troubleshooting](docs/usage.md#troubleshooting): help with connections, downloads and imports.
 - [Limitations](docs/usage.md#limitations): what magnetowid can't do.

cmd/magnetowid/main.go

@@ -2,6 +2,7 @@
 package main
 
 import (
+	"cmp"
 	"context"
 	"errors"
 	"flag"
@@ -45,13 +46,21 @@ func main() {
 
 func run(log *slog.Logger, level *slog.LevelVar) error {
 	listen := flag.String("listen", envOr("MAGNETOWID_LISTEN", ":8484"), "listen address")
-	apiKey := flag.String("api-key", os.Getenv("MAGNETOWID_API_KEY"), "API key for both APIs (required)")
+	// Usage prints flag defaults, so settings that hold secrets, the API key and
+	// a proxy URL's password, get theirs from the environment after parsing.
+	apiKey := flag.String("api-key", "", "API key for both APIs (required)")
 	downloadDir := flag.String("download-dir", os.Getenv("MAGNETOWID_DOWNLOAD_DIR"), "where finished downloads go (required)")
 	categories := flag.String("categories", envOr("MAGNETOWID_CATEGORIES", "tv,movies"), "comma-separated download categories")
 	ffmpeg := flag.String("ffmpeg", envOr("MAGNETOWID_FFMPEG", "ffmpeg"), "ffmpeg binary")
 	logLevel := flag.String("log-level", envOr("MAGNETOWID_LOG_LEVEL", "info"), "log level: debug, info, warn or error")
+	const proxyUsage = `'s HTTP proxy, e.g. http://127.0.0.1:8888, or "direct" for none (default: the environment's)`
+	tvpProxy := flag.String("tvp-proxy", "", "TVP VOD"+proxyUsage)
+	bbcProxy := flag.String("bbc-proxy", "", "BBC iPlayer"+proxyUsage)
 	healthcheck := flag.Bool("healthcheck", false, "ask the magnetowid at the listen address whether it is healthy, and exit")
 	flag.Parse()
+	*apiKey = cmp.Or(*apiKey, os.Getenv("MAGNETOWID_API_KEY"))
+	*tvpProxy = cmp.Or(*tvpProxy, os.Getenv("MAGNETOWID_TVP_PROXY"))
+	*bbcProxy = cmp.Or(*bbcProxy, os.Getenv("MAGNETOWID_BBC_PROXY"))
 
 	if *healthcheck {
 		return checkHealth(*listen)
@@ -88,12 +97,20 @@ func run(log *slog.Logger, level *slog.LevelVar) error {
 	}
 	defer db.Close()
 
-	httpClient := &http.Client{Timeout: 30 * time.Second}
-	tvpProvider, err := tvp.New(httpClient, log, db)
+	// Each site streams to its own country, so each has its own connection.
+	tvpClient, err := siteClient(*tvpProxy)
+	if err != nil {
+		return fmt.Errorf("-tvp-proxy: %w", err)
+	}
+	bbcClient, err := siteClient(*bbcProxy)
+	if err != nil {
+		return fmt.Errorf("-bbc-proxy: %w", err)
+	}
+	tvpProvider, err := tvp.New(tvpClient, log, db)
 	if err != nil {
 		return err
 	}
-	bbcProvider, err := bbc.New(httpClient, log, db)
+	bbcProvider, err := bbc.New(bbcClient, log, db)
 	if err != nil {
 		return err
 	}
@@ -111,7 +128,8 @@ func run(log *slog.Logger, level *slog.LevelVar) error {
 		return err
 	}
 	mux := http.NewServeMux()
-	prober := &probe.Prober{Client: httpClient}
+	// Streams bring their site's connection.
+	prober := &probe.Prober{Client: &http.Client{Timeout: siteTimeout}}
 	mux.Handle("/{provider}/api", &newznab.Handler{Providers: providers, APIKey: *apiKey, Probe: prober, Log: log})
 	mux.Handle("/api", &sabnzbd.Handler{Queue: queue, APIKey: *apiKey, Categories: cats, Log: log})
 	(&overrides.Handler{Store: overrideStore, APIKey: *apiKey, Log: log}).Register(mux)
@@ -130,7 +148,8 @@ func run(log *slog.Logger, level *slog.LevelVar) error {
 	srv := &http.Server{Addr: *listen, Handler: logRequests(log, mux), ReadHeaderTimeout: 10 * time.Second}
 	errc := make(chan error, 1)
 	go func() { errc <- srv.ListenAndServe() }()
-	log.Info("magnetowid listening", "version", version, "addr", *listen, "download_dir", dir, "providers", providers.Names())
+	log.Info("magnetowid listening", "version", version, "addr", *listen, "download_dir", dir, "providers", providers.Names(),
+		"tvp_proxy", proxyName(*tvpProxy), "bbc_proxy", proxyName(*bbcProxy))
 
 	var serveErr error
 	select {
@@ -168,6 +187,53 @@ func logRequests(log *slog.Logger, next http.Handler) http.Handler {
 	})
 }
 
+const (
+	siteTimeout = 30 * time.Second
+	// Skips the environment's proxy.
+	directProxy = "direct"
+)
+
+// siteClient returns the client for a site's API and streams. proxy is an HTTP
+// proxy's URL or "direct". Left empty, the client has no transport of its own:
+// the site and its streams follow the environment's HTTPS_PROXY and NO_PROXY,
+// which choose by host, and ffmpeg applies the environment's rules itself.
+func siteClient(proxy string) (*http.Client, error) {
+	client := &http.Client{Timeout: siteTimeout}
+	if proxy == "" {
+		return client, nil
+	}
+	t := http.DefaultTransport.(*http.Transport).Clone()
+	// Keep a connection for each of a download's segment workers.
+	t.MaxIdleConnsPerHost = 4
+	t.Proxy = nil
+	if proxy != directProxy {
+		// ffmpeg, which fetches some streams itself, tunnels through nothing else.
+		u, err := url.Parse(proxy)
+		if err != nil || u.Scheme != "http" || u.Host == "" {
+			// The URL may hold a password.
+			return nil, fmt.Errorf("want an http:// proxy URL or %q", directProxy)
+		}
+		t.Proxy = http.ProxyURL(u)
+	}
+	client.Transport = t
+	return client, nil
+}
+
+// proxyName describes a valid proxy setting for the log, without its password.
+func proxyName(proxy string) string {
+	switch proxy {
+	case "":
+		return "environment"
+	case directProxy:
+		return proxy
+	}
+	u, err := url.Parse(proxy)
+	if err != nil {
+		return "invalid"
+	}
+	return u.Redacted()
+}
+
 func checkHealth(listen string) error {
 	u, err := healthURL(listen)
 	if err != nil {

cmd/magnetowid/main_test.go

@@ -1,6 +1,15 @@
 package main
 
-import "testing"
+import (
+	"fmt"
+	"log/slog"
+	"net/http"
+	"os"
+	"os/exec"
+	"slices"
+	"strings"
+	"testing"
+)
 
 func TestHealthURL(t *testing.T) {
 	for listen, want := range map[string]string{
@@ -20,3 +29,93 @@ func TestHealthURL(t *testing.T) {
 		t.Error("healthURL accepted a listen address without a port")
 	}
 }
+
+// Usage, printed when a required setting is missing, lists every flag's
+// default and ends up in the service's log. The flags belong to the process,
+// so a second copy of the test binary runs with secrets in its environment.
+func TestUsageHidesSecrets(t *testing.T) {
+	if os.Getenv("MAGNETOWID_TEST_USAGE") != "" {
+		err := run(slog.New(slog.DiscardHandler), new(slog.LevelVar))
+		fmt.Fprintln(os.Stderr, "run:", err)
+		return
+	}
+	cmd := exec.Command(os.Args[0], "-test.run=^TestUsageHidesSecrets$")
+	cmd.Env = append(slices.DeleteFunc(os.Environ(), func(kv string) bool { return strings.HasPrefix(kv, "MAGNETOWID_") }),
+		"MAGNETOWID_TEST_USAGE=1",
+		"MAGNETOWID_API_KEY=key-secret",
+		"MAGNETOWID_TVP_PROXY=http://user:tvp-secret@proxy.example:3128",
+		"MAGNETOWID_BBC_PROXY=http://user:bbc-secret@proxy.example:3128")
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Fatalf("%v\n%s", err, out)
+	}
+	usage := string(out)
+	// The download folder is missing, so the key was read and usage printed.
+	if !strings.Contains(usage, "-tvp-proxy") || !strings.Contains(usage, "run: -api-key and -download-dir are required") {
+		t.Fatalf("no usage in the output:\n%s", usage)
+	}
+	if strings.Contains(usage, "secret") {
+		t.Errorf("usage shows a secret:\n%s", usage)
+	}
+}
+
+func TestSiteClient(t *testing.T) {
+	req, err := http.NewRequest(http.MethodGet, "https://vod.example/playlist.m3u8", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	transport := func(proxy string) *http.Transport {
+		t.Helper()
+		c, err := siteClient(proxy)
+		if err != nil {
+			t.Fatalf("siteClient(%q): %v", proxy, err)
+		}
+		return c.Transport.(*http.Transport)
+	}
+
+	// Unset, the environment decides host by host, for ffmpeg too: a transport
+	// here would make the downloader settle ffmpeg's proxy from one URL.
+	if c, err := siteClient(""); err != nil || c.Transport != nil || c.Timeout == 0 {
+		t.Errorf("siteClient(\"\") = %+v, %v; want a client without a transport", c, err)
+	}
+	if transport("direct").Proxy != nil {
+		t.Error(`"direct" still uses a proxy`)
+	}
+	for proxy, name := range map[string]string{
+		"http://proxy.example:3128":             "http://proxy.example:3128",
+		"http://user:secret@proxy.example:3128": "http://user:xxxxx@proxy.example:3128",
+	} {
+		u, err := transport(proxy).Proxy(req)
+		if err != nil || u == nil || u.String() != proxy {
+			t.Errorf("siteClient(%q) proxies through %v, %v", proxy, u, err)
+		}
+		if got := proxyName(proxy); got != name {
+			t.Errorf("proxyName(%q) = %q, want %q", proxy, got, name)
+		}
+	}
+	if a, b := transport("direct"), transport("direct"); a == b {
+		t.Error("sites share a transport")
+	}
+
+	// ffmpeg tunnels through HTTP proxies only.
+	for _, proxy := range []string{
+		"socks5://user:secret@proxy.example:1080",
+		"https://proxy.example:3128",
+		"proxy.example:3128",
+		"http://",
+		"http://[proxy",
+		"DIRECT",
+	} {
+		_, err := siteClient(proxy)
+		if err == nil {
+			t.Errorf("siteClient(%q) accepted the proxy", proxy)
+		} else if strings.Contains(err.Error(), "secret") {
+			t.Errorf("siteClient(%q) reports the password: %v", proxy, err)
+		}
+	}
+	for proxy, want := range map[string]string{"": "environment", "direct": "direct"} {
+		if got := proxyName(proxy); got != want {
+			t.Errorf("proxyName(%q) = %q, want %q", proxy, got, want)
+		}
+	}
+}

cmd/magnetowid/package_smoke_test.go

@@ -61,7 +61,7 @@ chown builder ./*
 runuser -u builder -- makepkg --nodeps
 pacman -U --noconfirm magnetowid-bin-*.pkg.tar.zst`,
 		upgrade: "pacman -U --noconfirm /build/magnetowid-bin-*.pkg.tar.zst",
-		remove:  "systemctl disable --now magnetowid && pacman -R --noconfirm magnetowid-bin",
+		remove:  "systemctl disable --now magnetowid magnetowid-vpn@pl && pacman -R --noconfirm magnetowid-bin",
 		x264:    true,
 	},
 }
@@ -166,19 +166,63 @@ systemctl start magnetowid`)
 		t.Errorf("category folder is %s, want 775 magnetowid:media", got)
 	}
 
+	// A VPN exit runs Gluetun with its instance's settings. Docker is faked: a
+	// script that records its arguments, behind a unit that does nothing. Like
+	// Gluetun, its container exits with 1 when stopped.
+	c.sh(t, `printf '#!/bin/sh\necho "$@" >>/run/docker.args\nif [ "$1" = run ]; then trap "exit 1" TERM; sleep infinity & wait; fi\nexit 0\n' >/usr/bin/docker
+chmod +x /usr/bin/docker
+printf '[Service]\nType=oneshot\nRemainAfterExit=yes\nExecStart=/bin/true\n' >/run/systemd/system/docker.service
+install -m 600 /usr/share/magnetowid/vpn.env.example /etc/magnetowid/vpn-pl.env
+systemctl daemon-reload
+systemctl enable --now magnetowid-vpn@pl`)
+	// The example names no VPN provider, so the exit must fail without retrying.
+	c.await(t, "systemctl is-failed magnetowid-vpn@pl", "failed")
+	if got := c.sh(t, "systemctl show -p ExecMainStatus -p NRestarts magnetowid-vpn@pl | sort | tr '\\n' ' '"); got != "ExecMainStatus=78 NRestarts=0" {
+		t.Errorf("the VPN exit without a provider ended with %s", got)
+	}
+	if run, _ := c.try("grep '^run ' /run/docker.args"); run != "" {
+		t.Errorf("the VPN exit ran docker %s without a provider", run)
+	}
+	c.sh(t, `sed -i 's/^VPN_SERVICE_PROVIDER=$/VPN_SERVICE_PROVIDER=example/; s/^MAGNETOWID_VPN_PORT=.*/MAGNETOWID_VPN_PORT=8889/' /etc/magnetowid/vpn-pl.env
+systemctl restart magnetowid-vpn@pl`)
+	c.await(t, "systemctl is-active magnetowid-vpn@pl", "active")
+	run := c.sh(t, "grep '^run ' /run/docker.args")
+	// Gluetun's proxy stays on its own port, whatever the local one.
+	for _, want := range []string{"--name magnetowid-vpn-pl ", "--env-file /etc/magnetowid/vpn-pl.env ",
+		"--publish 127.0.0.1:8889:8888/tcp ", " qmcgaw/gluetun:"} {
+		if !strings.Contains(run, want) {
+			t.Errorf("the VPN exit ran docker %s, without %q", run, want)
+		}
+	}
+	// Stopping it is not a failure, and it comes back if it stops by itself.
+	c.sh(t, "systemctl stop magnetowid-vpn@pl")
+	if state, _ := c.try("systemctl is-active magnetowid-vpn@pl"); state != "inactive" {
+		t.Errorf("stopped VPN exit is %s, want inactive", state)
+	}
+	c.sh(t, "systemctl start magnetowid-vpn@pl && kill -TERM $(systemctl show -p MainPID --value magnetowid-vpn@pl)")
+	c.await(t, "systemctl show -p NRestarts --value magnetowid-vpn@pl", "1")
+	c.await(t, "systemctl is-active magnetowid-vpn@pl", "active")
+
 	pid := c.sh(t, "systemctl show -p MainPID --value magnetowid")
 	c.sh(t, d.upgrade)
 	if got := c.sh(t, "systemctl show -p MainPID --value magnetowid"); (got != pid) != d.restarts {
 		t.Errorf("upgrade changed the main PID from %s to %s, want a restart: %v", pid, got, d.restarts)
 	}
 	checkAPIs(ctx, t, base, apiKey, downloadDir)
+	if state, _ := c.try("systemctl is-active magnetowid-vpn@pl"); state != "active" {
+		t.Errorf("upgraded package's VPN exit is %s, want active", state)
+	}
 
 	c.sh(t, d.remove)
-	if state, _ := c.try("systemctl is-active magnetowid"); state != "inactive" {
-		t.Errorf("removed service is %s, want inactive", state)
+	for _, unit := range []string{"magnetowid", "magnetowid-vpn@pl"} {
+		if state, _ := c.try("systemctl is-active " + unit); state != "inactive" {
+			t.Errorf("removed package's %s is %s, want inactive", unit, state)
+		}
+		c.sh(t, "test ! -e /etc/systemd/system/multi-user.target.wants/"+unit+".service")
 	}
 	c.sh(t, "test ! -e /usr/lib/systemd/system/magnetowid.service")
-	c.sh(t, "test ! -e /etc/systemd/system/multi-user.target.wants/magnetowid.service")
+	c.sh(t, "test ! -e /usr/lib/systemd/system/magnetowid-vpn@.service")
+	c.sh(t, "test ! -e /usr/share/magnetowid/vpn.env.example")
 }
 
 type container struct {