diff --git a/.goreleaser.yml b/.goreleaser.yml index ada352f..7316c4d 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -63,6 +63,12 @@ archives: - src: packaging/linux/magnetowid.{service,sysusers,env} dst: systemd strip_parent: true + - src: packaging/linux/magnetowid-vpn@.service + dst: systemd + strip_parent: true + - src: vpn.env.example + dst: systemd + strip_parent: true checksum: name_template: '{{ .ProjectName }}_{{ .Version }}_SHA256SUMS' @@ -106,6 +112,13 @@ nfpms: dst: /usr/lib/systemd/system/magnetowid.service - src: packaging/linux/magnetowid.sysusers dst: /usr/lib/sysusers.d/magnetowid.conf + - src: packaging/linux/magnetowid-vpn@.service + dst: /usr/lib/systemd/system/magnetowid-vpn@.service + # Outside /usr/share/doc, which minimal installs leave out. + - dst: /usr/share/magnetowid + type: dir + - src: vpn.env.example + dst: /usr/share/magnetowid/vpn.env.example - dst: /etc/magnetowid type: dir - src: packaging/linux/magnetowid.env @@ -135,6 +148,8 @@ aurs: depends: - ca-certificates - ffmpeg + optdepends: + - 'docker: VPN exits for region-locked sites' backup: - etc/magnetowid/magnetowid.env install: packaging/linux/magnetowid.install @@ -142,7 +157,9 @@ aurs: install -Dm755 "./magnetowid" "${pkgdir}/usr/bin/magnetowid" install -Dm644 "./LICENSE" "${pkgdir}/usr/share/licenses/magnetowid-bin/LICENSE" install -Dm644 "./systemd/magnetowid.service" "${pkgdir}/usr/lib/systemd/system/magnetowid.service" + install -Dm644 "./systemd/magnetowid-vpn@.service" "${pkgdir}/usr/lib/systemd/system/magnetowid-vpn@.service" install -Dm644 "./systemd/magnetowid.sysusers" "${pkgdir}/usr/lib/sysusers.d/magnetowid.conf" + install -Dm644 "./systemd/vpn.env.example" "${pkgdir}/usr/share/magnetowid/vpn.env.example" install -Dm600 "./systemd/magnetowid.env" "${pkgdir}/etc/magnetowid/magnetowid.env" homebrew_casks: diff --git a/README.md b/README.md index 9cb580c..4acedea 100644 --- a/README.md +++ b/README.md @@ -44,11 +44,12 @@ itself. No separate SABnzbd installation is needed. | Site | Indexer | Good to know | |---|---|---| -| **TVP VOD** | `/tvp` | Polish series and films. [TVP VOD notes](internal/provider/tvp/README.md) | +| **TVP VOD** | `/tvp` | Polish series and films. Streams may need a Polish connection. [TVP VOD notes](internal/provider/tvp/README.md) | | **BBC iPlayer** | `/bbc` | Streams need a UK connection. [BBC iPlayer notes](internal/provider/bbc/README.md) | Only what a site offers for free is found: DRM-protected, paid and -region-blocked streams are left out of the results. +region-blocked streams are left out of the results. To use a site from another +country, give it a VPN exit: see [Region-locked sites](docs/vpn.md). ## What you get @@ -156,6 +157,7 @@ See [Web interface](docs/usage.md#web-interface) for the details. ## Documentation - [Setup and configuration](docs/usage.md): installation options and every setting. +- [Region-locked sites](docs/vpn.md): a VPN exit for each site's country. - [Correcting matches](docs/usage.md#correcting-matches): overrides for titles, seasons and single episodes. - [Troubleshooting](docs/usage.md#troubleshooting): help with connections, downloads and imports. - [Limitations](docs/usage.md#limitations): what magnetowid can't do. diff --git a/cmd/magnetowid/main.go b/cmd/magnetowid/main.go index 6d80baa..42388dd 100644 --- a/cmd/magnetowid/main.go +++ b/cmd/magnetowid/main.go @@ -2,6 +2,7 @@ package main import ( + "cmp" "context" "errors" "flag" @@ -45,13 +46,21 @@ func main() { func run(log *slog.Logger, level *slog.LevelVar) error { listen := flag.String("listen", envOr("MAGNETOWID_LISTEN", ":8484"), "listen address") - apiKey := flag.String("api-key", os.Getenv("MAGNETOWID_API_KEY"), "API key for both APIs (required)") + // Usage prints flag defaults, so settings that hold secrets, the API key and + // a proxy URL's password, get theirs from the environment after parsing. + apiKey := flag.String("api-key", "", "API key for both APIs (required)") downloadDir := flag.String("download-dir", os.Getenv("MAGNETOWID_DOWNLOAD_DIR"), "where finished downloads go (required)") categories := flag.String("categories", envOr("MAGNETOWID_CATEGORIES", "tv,movies"), "comma-separated download categories") ffmpeg := flag.String("ffmpeg", envOr("MAGNETOWID_FFMPEG", "ffmpeg"), "ffmpeg binary") logLevel := flag.String("log-level", envOr("MAGNETOWID_LOG_LEVEL", "info"), "log level: debug, info, warn or error") + const proxyUsage = `'s HTTP proxy, e.g. http://127.0.0.1:8888, or "direct" for none (default: the environment's)` + tvpProxy := flag.String("tvp-proxy", "", "TVP VOD"+proxyUsage) + bbcProxy := flag.String("bbc-proxy", "", "BBC iPlayer"+proxyUsage) healthcheck := flag.Bool("healthcheck", false, "ask the magnetowid at the listen address whether it is healthy, and exit") flag.Parse() + *apiKey = cmp.Or(*apiKey, os.Getenv("MAGNETOWID_API_KEY")) + *tvpProxy = cmp.Or(*tvpProxy, os.Getenv("MAGNETOWID_TVP_PROXY")) + *bbcProxy = cmp.Or(*bbcProxy, os.Getenv("MAGNETOWID_BBC_PROXY")) if *healthcheck { return checkHealth(*listen) @@ -88,12 +97,20 @@ func run(log *slog.Logger, level *slog.LevelVar) error { } defer db.Close() - httpClient := &http.Client{Timeout: 30 * time.Second} - tvpProvider, err := tvp.New(httpClient, log, db) + // Each site streams to its own country, so each has its own connection. + tvpClient, err := siteClient(*tvpProxy) + if err != nil { + return fmt.Errorf("-tvp-proxy: %w", err) + } + bbcClient, err := siteClient(*bbcProxy) + if err != nil { + return fmt.Errorf("-bbc-proxy: %w", err) + } + tvpProvider, err := tvp.New(tvpClient, log, db) if err != nil { return err } - bbcProvider, err := bbc.New(httpClient, log, db) + bbcProvider, err := bbc.New(bbcClient, log, db) if err != nil { return err } @@ -111,7 +128,8 @@ func run(log *slog.Logger, level *slog.LevelVar) error { return err } mux := http.NewServeMux() - prober := &probe.Prober{Client: httpClient} + // Streams bring their site's connection. + prober := &probe.Prober{Client: &http.Client{Timeout: siteTimeout}} mux.Handle("/{provider}/api", &newznab.Handler{Providers: providers, APIKey: *apiKey, Probe: prober, Log: log}) mux.Handle("/api", &sabnzbd.Handler{Queue: queue, APIKey: *apiKey, Categories: cats, Log: log}) (&overrides.Handler{Store: overrideStore, APIKey: *apiKey, Log: log}).Register(mux) @@ -130,7 +148,8 @@ func run(log *slog.Logger, level *slog.LevelVar) error { srv := &http.Server{Addr: *listen, Handler: logRequests(log, mux), ReadHeaderTimeout: 10 * time.Second} errc := make(chan error, 1) go func() { errc <- srv.ListenAndServe() }() - log.Info("magnetowid listening", "version", version, "addr", *listen, "download_dir", dir, "providers", providers.Names()) + log.Info("magnetowid listening", "version", version, "addr", *listen, "download_dir", dir, "providers", providers.Names(), + "tvp_proxy", proxyName(*tvpProxy), "bbc_proxy", proxyName(*bbcProxy)) var serveErr error select { @@ -168,6 +187,53 @@ func logRequests(log *slog.Logger, next http.Handler) http.Handler { }) } +const ( + siteTimeout = 30 * time.Second + // Skips the environment's proxy. + directProxy = "direct" +) + +// siteClient returns the client for a site's API and streams. proxy is an HTTP +// proxy's URL or "direct". Left empty, the client has no transport of its own: +// the site and its streams follow the environment's HTTPS_PROXY and NO_PROXY, +// which choose by host, and ffmpeg applies the environment's rules itself. +func siteClient(proxy string) (*http.Client, error) { + client := &http.Client{Timeout: siteTimeout} + if proxy == "" { + return client, nil + } + t := http.DefaultTransport.(*http.Transport).Clone() + // Keep a connection for each of a download's segment workers. + t.MaxIdleConnsPerHost = 4 + t.Proxy = nil + if proxy != directProxy { + // ffmpeg, which fetches some streams itself, tunnels through nothing else. + u, err := url.Parse(proxy) + if err != nil || u.Scheme != "http" || u.Host == "" { + // The URL may hold a password. + return nil, fmt.Errorf("want an http:// proxy URL or %q", directProxy) + } + t.Proxy = http.ProxyURL(u) + } + client.Transport = t + return client, nil +} + +// proxyName describes a valid proxy setting for the log, without its password. +func proxyName(proxy string) string { + switch proxy { + case "": + return "environment" + case directProxy: + return proxy + } + u, err := url.Parse(proxy) + if err != nil { + return "invalid" + } + return u.Redacted() +} + func checkHealth(listen string) error { u, err := healthURL(listen) if err != nil { diff --git a/cmd/magnetowid/main_test.go b/cmd/magnetowid/main_test.go index 0874408..a95162b 100644 --- a/cmd/magnetowid/main_test.go +++ b/cmd/magnetowid/main_test.go @@ -1,6 +1,15 @@ package main -import "testing" +import ( + "fmt" + "log/slog" + "net/http" + "os" + "os/exec" + "slices" + "strings" + "testing" +) func TestHealthURL(t *testing.T) { for listen, want := range map[string]string{ @@ -20,3 +29,93 @@ func TestHealthURL(t *testing.T) { t.Error("healthURL accepted a listen address without a port") } } + +// Usage, printed when a required setting is missing, lists every flag's +// default and ends up in the service's log. The flags belong to the process, +// so a second copy of the test binary runs with secrets in its environment. +func TestUsageHidesSecrets(t *testing.T) { + if os.Getenv("MAGNETOWID_TEST_USAGE") != "" { + err := run(slog.New(slog.DiscardHandler), new(slog.LevelVar)) + fmt.Fprintln(os.Stderr, "run:", err) + return + } + cmd := exec.Command(os.Args[0], "-test.run=^TestUsageHidesSecrets$") + cmd.Env = append(slices.DeleteFunc(os.Environ(), func(kv string) bool { return strings.HasPrefix(kv, "MAGNETOWID_") }), + "MAGNETOWID_TEST_USAGE=1", + "MAGNETOWID_API_KEY=key-secret", + "MAGNETOWID_TVP_PROXY=http://user:tvp-secret@proxy.example:3128", + "MAGNETOWID_BBC_PROXY=http://user:bbc-secret@proxy.example:3128") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("%v\n%s", err, out) + } + usage := string(out) + // The download folder is missing, so the key was read and usage printed. + if !strings.Contains(usage, "-tvp-proxy") || !strings.Contains(usage, "run: -api-key and -download-dir are required") { + t.Fatalf("no usage in the output:\n%s", usage) + } + if strings.Contains(usage, "secret") { + t.Errorf("usage shows a secret:\n%s", usage) + } +} + +func TestSiteClient(t *testing.T) { + req, err := http.NewRequest(http.MethodGet, "https://vod.example/playlist.m3u8", nil) + if err != nil { + t.Fatal(err) + } + transport := func(proxy string) *http.Transport { + t.Helper() + c, err := siteClient(proxy) + if err != nil { + t.Fatalf("siteClient(%q): %v", proxy, err) + } + return c.Transport.(*http.Transport) + } + + // Unset, the environment decides host by host, for ffmpeg too: a transport + // here would make the downloader settle ffmpeg's proxy from one URL. + if c, err := siteClient(""); err != nil || c.Transport != nil || c.Timeout == 0 { + t.Errorf("siteClient(\"\") = %+v, %v; want a client without a transport", c, err) + } + if transport("direct").Proxy != nil { + t.Error(`"direct" still uses a proxy`) + } + for proxy, name := range map[string]string{ + "http://proxy.example:3128": "http://proxy.example:3128", + "http://user:secret@proxy.example:3128": "http://user:xxxxx@proxy.example:3128", + } { + u, err := transport(proxy).Proxy(req) + if err != nil || u == nil || u.String() != proxy { + t.Errorf("siteClient(%q) proxies through %v, %v", proxy, u, err) + } + if got := proxyName(proxy); got != name { + t.Errorf("proxyName(%q) = %q, want %q", proxy, got, name) + } + } + if a, b := transport("direct"), transport("direct"); a == b { + t.Error("sites share a transport") + } + + // ffmpeg tunnels through HTTP proxies only. + for _, proxy := range []string{ + "socks5://user:secret@proxy.example:1080", + "https://proxy.example:3128", + "proxy.example:3128", + "http://", + "http://[proxy", + "DIRECT", + } { + _, err := siteClient(proxy) + if err == nil { + t.Errorf("siteClient(%q) accepted the proxy", proxy) + } else if strings.Contains(err.Error(), "secret") { + t.Errorf("siteClient(%q) reports the password: %v", proxy, err) + } + } + for proxy, want := range map[string]string{"": "environment", "direct": "direct"} { + if got := proxyName(proxy); got != want { + t.Errorf("proxyName(%q) = %q, want %q", proxy, got, want) + } + } +} diff --git a/cmd/magnetowid/package_smoke_test.go b/cmd/magnetowid/package_smoke_test.go index a264f50..fb84738 100644 --- a/cmd/magnetowid/package_smoke_test.go +++ b/cmd/magnetowid/package_smoke_test.go @@ -61,7 +61,7 @@ chown builder ./* runuser -u builder -- makepkg --nodeps pacman -U --noconfirm magnetowid-bin-*.pkg.tar.zst`, upgrade: "pacman -U --noconfirm /build/magnetowid-bin-*.pkg.tar.zst", - remove: "systemctl disable --now magnetowid && pacman -R --noconfirm magnetowid-bin", + remove: "systemctl disable --now magnetowid magnetowid-vpn@pl && pacman -R --noconfirm magnetowid-bin", x264: true, }, } @@ -166,19 +166,63 @@ systemctl start magnetowid`) t.Errorf("category folder is %s, want 775 magnetowid:media", got) } + // A VPN exit runs Gluetun with its instance's settings. Docker is faked: a + // script that records its arguments, behind a unit that does nothing. Like + // Gluetun, its container exits with 1 when stopped. + c.sh(t, `printf '#!/bin/sh\necho "$@" >>/run/docker.args\nif [ "$1" = run ]; then trap "exit 1" TERM; sleep infinity & wait; fi\nexit 0\n' >/usr/bin/docker +chmod +x /usr/bin/docker +printf '[Service]\nType=oneshot\nRemainAfterExit=yes\nExecStart=/bin/true\n' >/run/systemd/system/docker.service +install -m 600 /usr/share/magnetowid/vpn.env.example /etc/magnetowid/vpn-pl.env +systemctl daemon-reload +systemctl enable --now magnetowid-vpn@pl`) + // The example names no VPN provider, so the exit must fail without retrying. + c.await(t, "systemctl is-failed magnetowid-vpn@pl", "failed") + if got := c.sh(t, "systemctl show -p ExecMainStatus -p NRestarts magnetowid-vpn@pl | sort | tr '\\n' ' '"); got != "ExecMainStatus=78 NRestarts=0" { + t.Errorf("the VPN exit without a provider ended with %s", got) + } + if run, _ := c.try("grep '^run ' /run/docker.args"); run != "" { + t.Errorf("the VPN exit ran docker %s without a provider", run) + } + c.sh(t, `sed -i 's/^VPN_SERVICE_PROVIDER=$/VPN_SERVICE_PROVIDER=example/; s/^MAGNETOWID_VPN_PORT=.*/MAGNETOWID_VPN_PORT=8889/' /etc/magnetowid/vpn-pl.env +systemctl restart magnetowid-vpn@pl`) + c.await(t, "systemctl is-active magnetowid-vpn@pl", "active") + run := c.sh(t, "grep '^run ' /run/docker.args") + // Gluetun's proxy stays on its own port, whatever the local one. + for _, want := range []string{"--name magnetowid-vpn-pl ", "--env-file /etc/magnetowid/vpn-pl.env ", + "--publish 127.0.0.1:8889:8888/tcp ", " qmcgaw/gluetun:"} { + if !strings.Contains(run, want) { + t.Errorf("the VPN exit ran docker %s, without %q", run, want) + } + } + // Stopping it is not a failure, and it comes back if it stops by itself. + c.sh(t, "systemctl stop magnetowid-vpn@pl") + if state, _ := c.try("systemctl is-active magnetowid-vpn@pl"); state != "inactive" { + t.Errorf("stopped VPN exit is %s, want inactive", state) + } + c.sh(t, "systemctl start magnetowid-vpn@pl && kill -TERM $(systemctl show -p MainPID --value magnetowid-vpn@pl)") + c.await(t, "systemctl show -p NRestarts --value magnetowid-vpn@pl", "1") + c.await(t, "systemctl is-active magnetowid-vpn@pl", "active") + pid := c.sh(t, "systemctl show -p MainPID --value magnetowid") c.sh(t, d.upgrade) if got := c.sh(t, "systemctl show -p MainPID --value magnetowid"); (got != pid) != d.restarts { t.Errorf("upgrade changed the main PID from %s to %s, want a restart: %v", pid, got, d.restarts) } checkAPIs(ctx, t, base, apiKey, downloadDir) + if state, _ := c.try("systemctl is-active magnetowid-vpn@pl"); state != "active" { + t.Errorf("upgraded package's VPN exit is %s, want active", state) + } c.sh(t, d.remove) - if state, _ := c.try("systemctl is-active magnetowid"); state != "inactive" { - t.Errorf("removed service is %s, want inactive", state) + for _, unit := range []string{"magnetowid", "magnetowid-vpn@pl"} { + if state, _ := c.try("systemctl is-active " + unit); state != "inactive" { + t.Errorf("removed package's %s is %s, want inactive", unit, state) + } + c.sh(t, "test ! -e /etc/systemd/system/multi-user.target.wants/"+unit+".service") } c.sh(t, "test ! -e /usr/lib/systemd/system/magnetowid.service") - c.sh(t, "test ! -e /etc/systemd/system/multi-user.target.wants/magnetowid.service") + c.sh(t, "test ! -e /usr/lib/systemd/system/magnetowid-vpn@.service") + c.sh(t, "test ! -e /usr/share/magnetowid/vpn.env.example") } type container struct { diff --git a/compose.vpn.yaml b/compose.vpn.yaml new file mode 100644 index 0000000..6b157c3 --- /dev/null +++ b/compose.vpn.yaml @@ -0,0 +1,22 @@ +# Adds a VPN exit for a region-locked site to compose.yaml: +# +# docker compose -f compose.yaml -f compose.vpn.yaml up -d +# +# Guide: https://github.com/combor/magnetowid/blob/main/docs/vpn.md +services: + magnetowid: + environment: + # One setting per site, naming the exit in the site's country. + MAGNETOWID_TVP_PROXY: http://vpn-pl:8888 + depends_on: + - vpn-pl + + vpn-pl: + image: qmcgaw/gluetun:v3 + restart: unless-stopped + cap_add: + - NET_ADMIN + devices: + - /dev/net/tun:/dev/net/tun + # Copy vpn.env.example to vpn-pl.env and fill it in. + env_file: vpn-pl.env diff --git a/docs/usage.md b/docs/usage.md index 44bdac8..913384d 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -7,7 +7,9 @@ magnetowid exposes two APIs for Sonarr and Radarr: Supported sites: **TVP VOD** (`tvp`) and **BBC iPlayer** (`bbc`). See the [TVP VOD notes](../internal/provider/tvp/README.md) and the -[BBC iPlayer notes](../internal/provider/bbc/README.md). +[BBC iPlayer notes](../internal/provider/bbc/README.md). Sites limit their +streams to their own country; [Region-locked sites](vpn.md) shows how to use +them from elsewhere. ## Install and run @@ -65,6 +67,9 @@ Group=yourgroup Run `sudo systemctl restart magnetowid` after changing settings. +The packages also install `magnetowid-vpn@.service`, which stays unused until +you set up a VPN exit for a [region-locked site](vpn.md). + ### Build from source Requires Go 1.27.1+ and ffmpeg. @@ -94,6 +99,8 @@ section in `compose.yaml`. For the Linux service, set them in | `-categories` | `MAGNETOWID_CATEGORIES` | `tv,movies` | download categories to offer | | `-ffmpeg` | `MAGNETOWID_FFMPEG` | `ffmpeg` | ffmpeg binary | | `-log-level` | `MAGNETOWID_LOG_LEVEL` | `info` | `debug`, `info`, `warn`, or `error`; debug includes requests and RSS activity | +| `-tvp-proxy` | `MAGNETOWID_TVP_PROXY` | | TVP VOD's HTTP proxy, e.g. `http://127.0.0.1:8888`, or `direct`; see [Region-locked sites](vpn.md) | +| `-bbc-proxy` | `MAGNETOWID_BBC_PROXY` | | BBC iPlayer's HTTP proxy, or `direct`; see [Region-locked sites](vpn.md) | `GET /health` returns `OK` without an API key. The container health check runs `magnetowid -healthcheck`, which queries `MAGNETOWID_LISTEN` and exits with 0 @@ -291,6 +298,7 @@ the queue and history, with the last error of each failed or retrying job. | Downloads finish but are not imported | Mount the shared folder into Sonarr/Radarr, check file permissions and add a Remote Path Mapping if the paths differ. | | A magnetowid release is sent to another download client | Set the indexer's **Download Client** to `magnetowid`. | | Downloads stay queued with `provider unreachable` in the log | Check the network, DNS, and VPN. Jobs resume automatically when the site becomes reachable, without consuming retries. | +| One site finds nothing, or the log shows its results as `unavailable` | The site refuses streams outside its country. See [Region-locked sites](vpn.md). | | A series or film is missing, or its episodes are wrong | Check that the site has it for free, then add an [override](#correcting-matches). | To inspect recent container messages: diff --git a/docs/vpn.md b/docs/vpn.md new file mode 100644 index 0000000..21ecd88 --- /dev/null +++ b/docs/vpn.md @@ -0,0 +1,224 @@ +# Region-locked sites + +Sites limit their streams to their own country: + +| Site | Country | Setting | +|---|---|---| +| TVP VOD | Poland | `MAGNETOWID_TVP_PROXY` | +| BBC iPlayer | United Kingdom | `MAGNETOWID_BBC_PROXY` | + +A site in the country magnetowid runs in needs nothing. For any other site, +run a **VPN exit** in the site's country and name it in the site's setting. +magnetowid then reaches that site, and only that site, through the exit, so +one install can use sites from different countries. + +An exit is a [Gluetun](https://github.com/qdm12/gluetun) container connected +to your VPN provider. You need Docker and a VPN subscription with servers in +the countries you want. + +- [Linux service](#linux-service) +- [Docker Compose](#docker-compose) +- [The site settings](#the-site-settings) +- [Troubleshooting](#troubleshooting) + +## Linux service + +The packages include `magnetowid-vpn@.service`, which runs one exit per name. +The name is yours to choose; these steps use `pl` for an exit in Poland. + +### One site + +**1. Describe the exit.** Copy the example and edit the copy: + +```sh +sudo install -m 600 /usr/share/magnetowid/vpn.env.example /etc/magnetowid/vpn-pl.env +sudoedit /etc/magnetowid/vpn-pl.env +``` + +[The example](../vpn.env.example) looks like this: + +```sh +# The country this exit must be in. +SERVER_COUNTRIES=Poland + +# Your VPN provider and its credentials. Gluetun's wiki lists the lines each +# provider needs: https://github.com/qdm12/gluetun-wiki/tree/main/setup/providers +VPN_SERVICE_PROVIDER= +VPN_TYPE= + +# magnetowid reaches the exit through Gluetun's HTTP proxy. +HTTPPROXY=on + +# Linux service only: the local port of this exit's proxy. Give each exit its own. +MAGNETOWID_VPN_PORT=8888 +``` + +Fill in `VPN_SERVICE_PROVIDER` and `VPN_TYPE`, and add the credential lines +[your provider's page](https://github.com/qdm12/gluetun-wiki/tree/main/setup/providers) +lists. Leave the other lines as they are. + +**2. Start the exit**, now and at every boot: + +```sh +sudo systemctl enable --now magnetowid-vpn@pl +``` + +**3. Check its country.** The first start downloads Gluetun, so allow a minute: + +```sh +curl -x http://127.0.0.1:8888 https://ipinfo.io/country +``` + +This must print `PL`. If it doesn't, see [Troubleshooting](#troubleshooting). + +**4. Send the site through it.** Add to `/etc/magnetowid/magnetowid.env`: + +```sh +MAGNETOWID_TVP_PROXY=http://127.0.0.1:8888 +``` + +```sh +sudo systemctl restart magnetowid +``` + +### A second site + +Repeat the steps with another name, country and port. For BBC iPlayer: + +```sh +sudo install -m 600 /usr/share/magnetowid/vpn.env.example /etc/magnetowid/vpn-gb.env +sudoedit /etc/magnetowid/vpn-gb.env +``` + +In `vpn-gb.env`, fill in the provider lines as before and change two lines: + +```sh +SERVER_COUNTRIES=United Kingdom +MAGNETOWID_VPN_PORT=8889 +``` + +Your provider may want separate credentials, such as a second WireGuard key, +for a second connection at the same time. + +```sh +sudo systemctl enable --now magnetowid-vpn@gb +curl -x http://127.0.0.1:8889 https://ipinfo.io/country +``` + +This must print `GB`. `/etc/magnetowid/magnetowid.env` then names both exits: + +```sh +MAGNETOWID_TVP_PROXY=http://127.0.0.1:8888 +MAGNETOWID_BBC_PROXY=http://127.0.0.1:8889 +``` + +```sh +sudo systemctl restart magnetowid +``` + +## Docker Compose + +[`compose.vpn.yaml`](../compose.vpn.yaml) adds an exit in Poland, `vpn-pl`, to +the [quick start](../README.md#quick-start)'s `compose.yaml`. + +### One site + +**1. Get the files**, in the folder that holds `compose.yaml`: + +```sh +curl -fsSLO https://raw.githubusercontent.com/combor/magnetowid/main/compose.vpn.yaml +curl -fsSL https://raw.githubusercontent.com/combor/magnetowid/main/vpn.env.example -o vpn-pl.env +chmod 600 vpn-pl.env +``` + +**2. Describe the exit.** In `vpn-pl.env`, fill in `VPN_SERVICE_PROVIDER` and +`VPN_TYPE`, and add the credential lines +[your provider's page](https://github.com/qdm12/gluetun-wiki/tree/main/setup/providers) +lists. + +**3. Use both Compose files** from now on. Add to `.env`: + +```sh +COMPOSE_FILE=compose.yaml:compose.vpn.yaml +``` + +**4. Start, and check the exit's country:** + +```sh +docker compose up -d +docker compose exec vpn-pl wget -qO- https://ipinfo.io/country +``` + +This must print `PL`. TVP VOD already goes through the exit: +`compose.vpn.yaml` sets `MAGNETOWID_TVP_PROXY` to `http://vpn-pl:8888`. + +### A second site + +Copy `vpn-pl.env` to `vpn-gb.env`, set `SERVER_COUNTRIES=United Kingdom` in +the copy, and give it separate credentials if your provider wants them for a +second connection. Then add the exit and the site's setting to +`compose.vpn.yaml`: + +```yaml +services: + magnetowid: + environment: + MAGNETOWID_TVP_PROXY: http://vpn-pl:8888 + MAGNETOWID_BBC_PROXY: http://vpn-gb:8888 + depends_on: + - vpn-pl + - vpn-gb + + vpn-pl: + image: qmcgaw/gluetun:v3 + restart: unless-stopped + cap_add: + - NET_ADMIN + devices: + - /dev/net/tun:/dev/net/tun + env_file: vpn-pl.env + + vpn-gb: + image: qmcgaw/gluetun:v3 + restart: unless-stopped + cap_add: + - NET_ADMIN + devices: + - /dev/net/tun:/dev/net/tun + env_file: vpn-gb.env +``` + +```sh +docker compose up -d +docker compose exec vpn-gb wget -qO- https://ipinfo.io/country +``` + +This must print `GB`. + +## The site settings + +Each takes one of: + +| Value | The site is reached | +|---|---| +| `http://host:port` | through that HTTP proxy | +| `direct` | without a proxy, even if `HTTPS_PROXY` is set | +| not set | as before: through `HTTPS_PROXY` if it is set, otherwise directly | + +The command-line flags are `-tvp-proxy` and `-bbc-proxy`. A site's searches, +streams and subtitles all use its setting. magnetowid logs every site's setting +when it starts. + +While an exit is down, its site's downloads wait as `provider unreachable` +and resume by themselves. They are never sent without the exit. + +## Troubleshooting + +| Problem | What to check | +|---|---| +| The exit doesn't start | `systemctl status magnetowid-vpn@pl` and `journalctl -u magnetowid-vpn@pl -n 50`, or `docker compose logs vpn-pl`. A status of `78/CONFIG` means `VPN_SERVICE_PROVIDER` or `MAGNETOWID_VPN_PORT` is missing from the exit's file. Other errors come from Gluetun and name the setting it rejects: a missing credential, or a country your provider's Gluetun page spells differently. | +| Gluetun rejects the country although it is right | `VPN_SERVICE_PROVIDER` is blank, so Gluetun assumed a provider of its own. | +| The country check fails to connect | The exit is still starting, or the port isn't the exit's `MAGNETOWID_VPN_PORT`. | +| One site's results are empty | Its streams are refused: the log's search lines show `unavailable` above zero. Check that site's setting and its exit's country, then restart magnetowid. | +| Downloads wait as `provider unreachable` | The site's exit is down. Start it; the downloads resume. | +| A download fails with an ffmpeg `Input/output error` | magnetowid fetches streams itself where it can; live and encrypted ones are left to ffmpeg, which can't connect through Gluetun's proxy. These fail instead of bypassing the exit. | diff --git a/internal/downloader/ffmpeg.go b/internal/downloader/ffmpeg.go index d375409..973d2a0 100644 --- a/internal/downloader/ffmpeg.go +++ b/internal/downloader/ffmpeg.go @@ -13,6 +13,7 @@ import ( "os" "os/exec" "path/filepath" + "slices" "strconv" "strings" "sync" @@ -33,8 +34,9 @@ const playlistTimeout = 30 * time.Second // Neither transcodes. type FFmpeg struct { Path string // binary; "ffmpeg" if empty - // Fetches playlists and segments; nil uses a default. Each download gets - // its own copy, with a cookie jar if the client has none. + // Fetches playlists and segments, on the stream's Transport if it has one; + // nil uses a default. Each download gets its own copy, with a cookie jar + // if the client has none. Client *http.Client Log *slog.Logger // nil discards // Zero defaults to 5 minutes, allowing for +faststart's silent final pass. @@ -81,6 +83,7 @@ func (f *FFmpeg) Download(ctx context.Context, s provider.Stream, out string, pr c := *f.Client client = &c } + client = s.Client(client) if client.Jar == nil { // ffmpeg sends a playlist's cookies with its segments too. client.Jar, _ = cookiejar.New(nil) @@ -108,13 +111,52 @@ func (f *FFmpeg) Download(ctx context.Context, s provider.Stream, out string, pr if err := os.RemoveAll(dir); err != nil { return err } + env, err := proxyEnv(s) + if err != nil { + return err + } inputs := directInputs(s, m, master) var args []string for _, in := range inputs { args = append(args, headerArgs(s.Header)...) args = append(args, "-i", in) } - return f.run(ctx, "", append(args, outputArgs(len(inputs), m.AudioLanguage, out)...), progress) + return f.run(ctx, "", env, append(args, outputArgs(len(inputs), m.AudioLanguage, out)...), progress) +} + +// proxyEnv returns the environment in which ffmpeg reaches the stream as its +// Transport does, or nil, magnetowid's own, for a stream without one. ffmpeg +// reads http_proxy for HTTP and HTTPS alike. +// +// The proxy for the stream's URL then serves every host ffmpeg fetches from, +// renditions and segments included, so a Transport must use one proxy for all +// hosts. Proxies chosen by host, as the environment's are, need a stream +// without a Transport: ffmpeg applies the environment's rules itself. +func proxyEnv(s provider.Stream) ([]string, error) { + t, ok := s.Transport.(*http.Transport) + if !ok { + return nil, nil + } + env := slices.DeleteFunc(os.Environ(), func(kv string) bool { + k, _, _ := strings.Cut(kv, "=") + return strings.EqualFold(k, "http_proxy") || strings.EqualFold(k, "no_proxy") + }) + if t.Proxy == nil { + return env, nil + } + req, err := http.NewRequest(http.MethodGet, s.URL, nil) + if err != nil { + return nil, err + } + // A proxy that can't be determined must not become a direct connection. + proxy, err := t.Proxy(req) + if err != nil { + return nil, fmt.Errorf("finding the stream's proxy: %w", err) + } + if proxy != nil { + env = append(env, "http_proxy="+proxy.String()) + } + return env, nil } func (f *FFmpeg) log() *slog.Logger { @@ -175,8 +217,9 @@ func (f *FFmpeg) binary() (string, error) { return bin, nil } -// run runs ffmpeg in dir, or the current folder if dir is empty. -func (f *FFmpeg) run(parent context.Context, dir string, args []string, progress func(time.Duration, int64)) error { +// run runs ffmpeg in dir, or the current folder if dir is empty, with env, or +// magnetowid's environment if env is nil. +func (f *FFmpeg) run(parent context.Context, dir string, env, args []string, progress func(time.Duration, int64)) error { bin, err := f.binary() if err != nil { return err @@ -191,6 +234,7 @@ func (f *FFmpeg) run(parent context.Context, dir string, args []string, progress args = append([]string{"-nostdin", "-hide_banner", "-loglevel", "warning", "-y"}, args...) cmd := exec.CommandContext(ctx, bin, args...) cmd.Dir = dir + cmd.Env = env cmd.WaitDelay = 5 * time.Second // Isolate ffmpeg from terminal signals so magnetowid can requeue interrupted jobs. cmd.SysProcAttr = ownProcessGroup() diff --git a/internal/downloader/ffmpeg_unix_test.go b/internal/downloader/ffmpeg_unix_test.go index 3ba6bf8..73e373b 100644 --- a/internal/downloader/ffmpeg_unix_test.go +++ b/internal/downloader/ffmpeg_unix_test.go @@ -8,11 +8,13 @@ import ( "io" "net/http" "net/http/httptest" + "net/url" "os" "path/filepath" "slices" "strconv" "strings" + "sync" "syscall" "testing" "time" @@ -58,11 +60,13 @@ func TestFFmpegRunsInOwnProcessGroup(t *testing.T) { } } -// fakeFFmpeg records its arguments and folder in log and creates its output. +// fakeFFmpeg records its folder, arguments and proxy settings in log and +// creates its output. func fakeFFmpeg(t *testing.T, dir string) (bin, log string) { t.Helper() bin, log = filepath.Join(dir, "ffmpeg"), filepath.Join(dir, "args") - script := "#!/bin/sh\npwd > '" + log + "'\nfor a; do echo \"$a\"; out=$a; done >> '" + log + "'\n: > \"$out\"\n" + script := "#!/bin/sh\npwd > '" + log + "'\nfor a; do echo \"$a\"; out=$a; done >> '" + log + "'\n" + + "echo \"http_proxy=$http_proxy no_proxy=$no_proxy\" >> '" + log + "'\n: > \"$out\"\n" if err := os.WriteFile(bin, []byte(script), 0o755); err != nil { t.Fatal(err) } @@ -110,6 +114,97 @@ func TestFFmpegInputs(t *testing.T) { } } +// ffmpeg reads its proxy from the environment, which must match the stream's +// transport and not magnetowid's own settings. +func TestFFmpegFollowsStreamProxy(t *testing.T) { + dir := t.TempDir() + bin, log := fakeFFmpeg(t, dir) + t.Setenv("http_proxy", "http://environment.example:3128") + t.Setenv("no_proxy", "elsewhere.example") + proxy, err := url.Parse("http://proxy.example:3128") + if err != nil { + t.Fatal(err) + } + for _, tt := range []struct { + name string + transport http.RoundTripper + want string + }{ + {"no transport", nil, "http_proxy=http://environment.example:3128 no_proxy=elsewhere.example"}, + {"proxy", &http.Transport{Proxy: http.ProxyURL(proxy)}, "http_proxy=http://proxy.example:3128 no_proxy="}, + {"direct", &http.Transport{}, "http_proxy= no_proxy="}, + {"no proxy for the stream", &http.Transport{Proxy: func(*http.Request) (*url.URL, error) { return nil, nil }}, + "http_proxy= no_proxy="}, + } { + // Not a playlist, so ffmpeg fetches it. + s := provider.Stream{URL: "http://example.invalid/video.mp4", Transport: tt.transport} + if err := (&FFmpeg{Path: bin}).Download(context.Background(), s, filepath.Join(dir, "out.mp4"), + func(time.Duration, int64) {}); err != nil { + t.Fatalf("%s: %v", tt.name, err) + } + b, err := os.ReadFile(log) + if err != nil { + t.Fatal(err) + } + if lines := strings.Split(strings.TrimSpace(string(b)), "\n"); lines[len(lines)-1] != tt.want { + t.Errorf("%s: ffmpeg ran with %q, want %q", tt.name, lines[len(lines)-1], tt.want) + } + } + + // An unknown proxy must not become a direct connection. + broken := &http.Transport{Proxy: func(*http.Request) (*url.URL, error) { return nil, errors.New("bad proxy") }} + os.Remove(log) + err = (&FFmpeg{Path: bin}).Download(context.Background(), + provider.Stream{URL: "http://example.invalid/video.mp4", Transport: broken}, filepath.Join(dir, "out.mp4"), + func(time.Duration, int64) {}) + if err == nil || !strings.Contains(err.Error(), "bad proxy") { + t.Errorf("err = %v, want the proxy's", err) + } + if _, err := os.Stat(log); !os.IsNotExist(err) { + t.Errorf("ffmpeg ran without knowing its proxy: %v", err) + } +} + +// The stream's transport, e.g. its site's proxy, fetches playlists and segments. +func TestDownloadUsesStreamTransport(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/master.m3u8": + io.WriteString(w, "#EXTM3U\n#EXT-X-STREAM-INF:BANDWIDTH=1\nvod.m3u8\n") + case "/vod.m3u8": + io.WriteString(w, "#EXTM3U\n#EXTINF:4,\n1.ts\n#EXT-X-ENDLIST\n") + default: + io.WriteString(w, "segment data") + } + })) + defer srv.Close() + dir := t.TempDir() + bin, _ := fakeFFmpeg(t, dir) + var mu sync.Mutex + var paths []string + site := roundTripFunc(func(req *http.Request) (*http.Response, error) { + mu.Lock() + paths = append(paths, req.URL.Path) + mu.Unlock() + return http.DefaultTransport.RoundTrip(req) + }) + // The engine's own client must stay unused. + other := &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + t.Errorf("%s fetched without the stream's transport", req.URL.Path) + return nil, errors.New("wrong transport") + })} + err := (&FFmpeg{Path: bin, Client: other}).Download(context.Background(), + provider.Stream{URL: srv.URL + "/master.m3u8", Transport: site}, filepath.Join(dir, "out.mp4"), + func(time.Duration, int64) {}) + if err != nil { + t.Fatal(err) + } + slices.Sort(paths) + if want := []string{"/1.ts", "/master.m3u8", "/vod.m3u8"}; !slices.Equal(paths, want) { + t.Errorf("the stream's transport fetched %q, want %q", paths, want) + } +} + // ffmpeg 5.x reports unreadable inputs without "Error opening input". func TestUnreadableSegmentsRemoved(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/internal/downloader/queue.go b/internal/downloader/queue.go index ea53ed3..709a4ac 100644 --- a/internal/downloader/queue.go +++ b/internal/downloader/queue.go @@ -650,7 +650,7 @@ func (q *Queue) saveSubtitles(ctx context.Context, j Job, s provider.Stream, wor if slices.Contains(paths, path) { continue // more in the same language } - srt, err := subtitles.Fetch(ctx, defaultClient, sub, s.Header) + srt, err := subtitles.Fetch(ctx, s.Client(defaultClient), sub, s.Header) if err == nil { err = os.WriteFile(path, srt, 0o666) } diff --git a/internal/downloader/queue_test.go b/internal/downloader/queue_test.go index ec27732..e9dc1af 100644 --- a/internal/downloader/queue_test.go +++ b/internal/downloader/queue_test.go @@ -15,6 +15,7 @@ import ( "slices" "strings" "sync" + "sync/atomic" "testing" "time" @@ -29,8 +30,13 @@ type fakeProvider struct { offline int err error subtitles []provider.Subtitle + transport http.RoundTripper } +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) } + func (f *fakeProvider) Name() string { return "fake" } func (f *fakeProvider) Search(context.Context, provider.Query) ([]provider.Item, error) { @@ -48,7 +54,8 @@ func (f *fakeProvider) Resolve(_ context.Context, id string) (provider.Stream, e if f.err != nil { return provider.Stream{}, f.err } - return provider.Stream{URL: fmt.Sprintf("http://example.invalid/%s/%d.m3u8", id, f.resolves), Subtitles: f.subtitles}, nil + return provider.Stream{URL: fmt.Sprintf("http://example.invalid/%s/%d.m3u8", id, f.resolves), Subtitles: f.subtitles, + Transport: f.transport}, nil } type fakeEngine struct { @@ -365,7 +372,13 @@ func TestSubtitlesSaved(t *testing.T) {
Dzień dobry.