diff --git a/.goreleaser.yml b/.goreleaser.yml index ada352f..7316c4d 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -63,6 +63,12 @@ archives: - src: packaging/linux/magnetowid.{service,sysusers,env} dst: systemd strip_parent: true + - src: packaging/linux/magnetowid-vpn@.service + dst: systemd + strip_parent: true + - src: vpn.env.example + dst: systemd + strip_parent: true checksum: name_template: '{{ .ProjectName }}_{{ .Version }}_SHA256SUMS' @@ -106,6 +112,13 @@ nfpms: dst: /usr/lib/systemd/system/magnetowid.service - src: packaging/linux/magnetowid.sysusers dst: /usr/lib/sysusers.d/magnetowid.conf + - src: packaging/linux/magnetowid-vpn@.service + dst: /usr/lib/systemd/system/magnetowid-vpn@.service + # Outside /usr/share/doc, which minimal installs leave out. + - dst: /usr/share/magnetowid + type: dir + - src: vpn.env.example + dst: /usr/share/magnetowid/vpn.env.example - dst: /etc/magnetowid type: dir - src: packaging/linux/magnetowid.env @@ -135,6 +148,8 @@ aurs: depends: - ca-certificates - ffmpeg + optdepends: + - 'docker: VPN exits for region-locked sites' backup: - etc/magnetowid/magnetowid.env install: packaging/linux/magnetowid.install @@ -142,7 +157,9 @@ aurs: install -Dm755 "./magnetowid" "${pkgdir}/usr/bin/magnetowid" install -Dm644 "./LICENSE" "${pkgdir}/usr/share/licenses/magnetowid-bin/LICENSE" install -Dm644 "./systemd/magnetowid.service" "${pkgdir}/usr/lib/systemd/system/magnetowid.service" + install -Dm644 "./systemd/magnetowid-vpn@.service" "${pkgdir}/usr/lib/systemd/system/magnetowid-vpn@.service" install -Dm644 "./systemd/magnetowid.sysusers" "${pkgdir}/usr/lib/sysusers.d/magnetowid.conf" + install -Dm644 "./systemd/vpn.env.example" "${pkgdir}/usr/share/magnetowid/vpn.env.example" install -Dm600 "./systemd/magnetowid.env" "${pkgdir}/etc/magnetowid/magnetowid.env" homebrew_casks: diff --git a/README.md b/README.md index 9cb580c..4acedea 100644 --- a/README.md +++ b/README.md @@ -44,11 +44,12 @@ itself. No separate SABnzbd installation is needed. | Site | Indexer | Good to know | |---|---|---| -| **TVP VOD** | `/tvp` | Polish series and films. [TVP VOD notes](internal/provider/tvp/README.md) | +| **TVP VOD** | `/tvp` | Polish series and films. Streams may need a Polish connection. [TVP VOD notes](internal/provider/tvp/README.md) | | **BBC iPlayer** | `/bbc` | Streams need a UK connection. [BBC iPlayer notes](internal/provider/bbc/README.md) | Only what a site offers for free is found: DRM-protected, paid and -region-blocked streams are left out of the results. +region-blocked streams are left out of the results. To use a site from another +country, give it a VPN exit: see [Region-locked sites](docs/vpn.md). ## What you get @@ -156,6 +157,7 @@ See [Web interface](docs/usage.md#web-interface) for the details. ## Documentation - [Setup and configuration](docs/usage.md): installation options and every setting. +- [Region-locked sites](docs/vpn.md): a VPN exit for each site's country. - [Correcting matches](docs/usage.md#correcting-matches): overrides for titles, seasons and single episodes. - [Troubleshooting](docs/usage.md#troubleshooting): help with connections, downloads and imports. - [Limitations](docs/usage.md#limitations): what magnetowid can't do. diff --git a/cmd/magnetowid/main.go b/cmd/magnetowid/main.go index 6d80baa..42388dd 100644 --- a/cmd/magnetowid/main.go +++ b/cmd/magnetowid/main.go @@ -2,6 +2,7 @@ package main import ( + "cmp" "context" "errors" "flag" @@ -45,13 +46,21 @@ func main() { func run(log *slog.Logger, level *slog.LevelVar) error { listen := flag.String("listen", envOr("MAGNETOWID_LISTEN", ":8484"), "listen address") - apiKey := flag.String("api-key", os.Getenv("MAGNETOWID_API_KEY"), "API key for both APIs (required)") + // Usage prints flag defaults, so settings that hold secrets, the API key and + // a proxy URL's password, get theirs from the environment after parsing. + apiKey := flag.String("api-key", "", "API key for both APIs (required)") downloadDir := flag.String("download-dir", os.Getenv("MAGNETOWID_DOWNLOAD_DIR"), "where finished downloads go (required)") categories := flag.String("categories", envOr("MAGNETOWID_CATEGORIES", "tv,movies"), "comma-separated download categories") ffmpeg := flag.String("ffmpeg", envOr("MAGNETOWID_FFMPEG", "ffmpeg"), "ffmpeg binary") logLevel := flag.String("log-level", envOr("MAGNETOWID_LOG_LEVEL", "info"), "log level: debug, info, warn or error") + const proxyUsage = `'s HTTP proxy, e.g. http://127.0.0.1:8888, or "direct" for none (default: the environment's)` + tvpProxy := flag.String("tvp-proxy", "", "TVP VOD"+proxyUsage) + bbcProxy := flag.String("bbc-proxy", "", "BBC iPlayer"+proxyUsage) healthcheck := flag.Bool("healthcheck", false, "ask the magnetowid at the listen address whether it is healthy, and exit") flag.Parse() + *apiKey = cmp.Or(*apiKey, os.Getenv("MAGNETOWID_API_KEY")) + *tvpProxy = cmp.Or(*tvpProxy, os.Getenv("MAGNETOWID_TVP_PROXY")) + *bbcProxy = cmp.Or(*bbcProxy, os.Getenv("MAGNETOWID_BBC_PROXY")) if *healthcheck { return checkHealth(*listen) @@ -88,12 +97,20 @@ func run(log *slog.Logger, level *slog.LevelVar) error { } defer db.Close() - httpClient := &http.Client{Timeout: 30 * time.Second} - tvpProvider, err := tvp.New(httpClient, log, db) + // Each site streams to its own country, so each has its own connection. + tvpClient, err := siteClient(*tvpProxy) + if err != nil { + return fmt.Errorf("-tvp-proxy: %w", err) + } + bbcClient, err := siteClient(*bbcProxy) + if err != nil { + return fmt.Errorf("-bbc-proxy: %w", err) + } + tvpProvider, err := tvp.New(tvpClient, log, db) if err != nil { return err } - bbcProvider, err := bbc.New(httpClient, log, db) + bbcProvider, err := bbc.New(bbcClient, log, db) if err != nil { return err } @@ -111,7 +128,8 @@ func run(log *slog.Logger, level *slog.LevelVar) error { return err } mux := http.NewServeMux() - prober := &probe.Prober{Client: httpClient} + // Streams bring their site's connection. + prober := &probe.Prober{Client: &http.Client{Timeout: siteTimeout}} mux.Handle("/{provider}/api", &newznab.Handler{Providers: providers, APIKey: *apiKey, Probe: prober, Log: log}) mux.Handle("/api", &sabnzbd.Handler{Queue: queue, APIKey: *apiKey, Categories: cats, Log: log}) (&overrides.Handler{Store: overrideStore, APIKey: *apiKey, Log: log}).Register(mux) @@ -130,7 +148,8 @@ func run(log *slog.Logger, level *slog.LevelVar) error { srv := &http.Server{Addr: *listen, Handler: logRequests(log, mux), ReadHeaderTimeout: 10 * time.Second} errc := make(chan error, 1) go func() { errc <- srv.ListenAndServe() }() - log.Info("magnetowid listening", "version", version, "addr", *listen, "download_dir", dir, "providers", providers.Names()) + log.Info("magnetowid listening", "version", version, "addr", *listen, "download_dir", dir, "providers", providers.Names(), + "tvp_proxy", proxyName(*tvpProxy), "bbc_proxy", proxyName(*bbcProxy)) var serveErr error select { @@ -168,6 +187,53 @@ func logRequests(log *slog.Logger, next http.Handler) http.Handler { }) } +const ( + siteTimeout = 30 * time.Second + // Skips the environment's proxy. + directProxy = "direct" +) + +// siteClient returns the client for a site's API and streams. proxy is an HTTP +// proxy's URL or "direct". Left empty, the client has no transport of its own: +// the site and its streams follow the environment's HTTPS_PROXY and NO_PROXY, +// which choose by host, and ffmpeg applies the environment's rules itself. +func siteClient(proxy string) (*http.Client, error) { + client := &http.Client{Timeout: siteTimeout} + if proxy == "" { + return client, nil + } + t := http.DefaultTransport.(*http.Transport).Clone() + // Keep a connection for each of a download's segment workers. + t.MaxIdleConnsPerHost = 4 + t.Proxy = nil + if proxy != directProxy { + // ffmpeg, which fetches some streams itself, tunnels through nothing else. + u, err := url.Parse(proxy) + if err != nil || u.Scheme != "http" || u.Host == "" { + // The URL may hold a password. + return nil, fmt.Errorf("want an http:// proxy URL or %q", directProxy) + } + t.Proxy = http.ProxyURL(u) + } + client.Transport = t + return client, nil +} + +// proxyName describes a valid proxy setting for the log, without its password. +func proxyName(proxy string) string { + switch proxy { + case "": + return "environment" + case directProxy: + return proxy + } + u, err := url.Parse(proxy) + if err != nil { + return "invalid" + } + return u.Redacted() +} + func checkHealth(listen string) error { u, err := healthURL(listen) if err != nil { diff --git a/cmd/magnetowid/main_test.go b/cmd/magnetowid/main_test.go index 0874408..a95162b 100644 --- a/cmd/magnetowid/main_test.go +++ b/cmd/magnetowid/main_test.go @@ -1,6 +1,15 @@ package main -import "testing" +import ( + "fmt" + "log/slog" + "net/http" + "os" + "os/exec" + "slices" + "strings" + "testing" +) func TestHealthURL(t *testing.T) { for listen, want := range map[string]string{ @@ -20,3 +29,93 @@ func TestHealthURL(t *testing.T) { t.Error("healthURL accepted a listen address without a port") } } + +// Usage, printed when a required setting is missing, lists every flag's +// default and ends up in the service's log. The flags belong to the process, +// so a second copy of the test binary runs with secrets in its environment. +func TestUsageHidesSecrets(t *testing.T) { + if os.Getenv("MAGNETOWID_TEST_USAGE") != "" { + err := run(slog.New(slog.DiscardHandler), new(slog.LevelVar)) + fmt.Fprintln(os.Stderr, "run:", err) + return + } + cmd := exec.Command(os.Args[0], "-test.run=^TestUsageHidesSecrets$") + cmd.Env = append(slices.DeleteFunc(os.Environ(), func(kv string) bool { return strings.HasPrefix(kv, "MAGNETOWID_") }), + "MAGNETOWID_TEST_USAGE=1", + "MAGNETOWID_API_KEY=key-secret", + "MAGNETOWID_TVP_PROXY=http://user:tvp-secret@proxy.example:3128", + "MAGNETOWID_BBC_PROXY=http://user:bbc-secret@proxy.example:3128") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("%v\n%s", err, out) + } + usage := string(out) + // The download folder is missing, so the key was read and usage printed. + if !strings.Contains(usage, "-tvp-proxy") || !strings.Contains(usage, "run: -api-key and -download-dir are required") { + t.Fatalf("no usage in the output:\n%s", usage) + } + if strings.Contains(usage, "secret") { + t.Errorf("usage shows a secret:\n%s", usage) + } +} + +func TestSiteClient(t *testing.T) { + req, err := http.NewRequest(http.MethodGet, "https://vod.example/playlist.m3u8", nil) + if err != nil { + t.Fatal(err) + } + transport := func(proxy string) *http.Transport { + t.Helper() + c, err := siteClient(proxy) + if err != nil { + t.Fatalf("siteClient(%q): %v", proxy, err) + } + return c.Transport.(*http.Transport) + } + + // Unset, the environment decides host by host, for ffmpeg too: a transport + // here would make the downloader settle ffmpeg's proxy from one URL. + if c, err := siteClient(""); err != nil || c.Transport != nil || c.Timeout == 0 { + t.Errorf("siteClient(\"\") = %+v, %v; want a client without a transport", c, err) + } + if transport("direct").Proxy != nil { + t.Error(`"direct" still uses a proxy`) + } + for proxy, name := range map[string]string{ + "http://proxy.example:3128": "http://proxy.example:3128", + "http://user:secret@proxy.example:3128": "http://user:xxxxx@proxy.example:3128", + } { + u, err := transport(proxy).Proxy(req) + if err != nil || u == nil || u.String() != proxy { + t.Errorf("siteClient(%q) proxies through %v, %v", proxy, u, err) + } + if got := proxyName(proxy); got != name { + t.Errorf("proxyName(%q) = %q, want %q", proxy, got, name) + } + } + if a, b := transport("direct"), transport("direct"); a == b { + t.Error("sites share a transport") + } + + // ffmpeg tunnels through HTTP proxies only. + for _, proxy := range []string{ + "socks5://user:secret@proxy.example:1080", + "https://proxy.example:3128", + "proxy.example:3128", + "http://", + "http://[proxy", + "DIRECT", + } { + _, err := siteClient(proxy) + if err == nil { + t.Errorf("siteClient(%q) accepted the proxy", proxy) + } else if strings.Contains(err.Error(), "secret") { + t.Errorf("siteClient(%q) reports the password: %v", proxy, err) + } + } + for proxy, want := range map[string]string{"": "environment", "direct": "direct"} { + if got := proxyName(proxy); got != want { + t.Errorf("proxyName(%q) = %q, want %q", proxy, got, want) + } + } +} diff --git a/cmd/magnetowid/package_smoke_test.go b/cmd/magnetowid/package_smoke_test.go index a264f50..fb84738 100644 --- a/cmd/magnetowid/package_smoke_test.go +++ b/cmd/magnetowid/package_smoke_test.go @@ -61,7 +61,7 @@ chown builder ./* runuser -u builder -- makepkg --nodeps pacman -U --noconfirm magnetowid-bin-*.pkg.tar.zst`, upgrade: "pacman -U --noconfirm /build/magnetowid-bin-*.pkg.tar.zst", - remove: "systemctl disable --now magnetowid && pacman -R --noconfirm magnetowid-bin", + remove: "systemctl disable --now magnetowid magnetowid-vpn@pl && pacman -R --noconfirm magnetowid-bin", x264: true, }, } @@ -166,19 +166,63 @@ systemctl start magnetowid`) t.Errorf("category folder is %s, want 775 magnetowid:media", got) } + // A VPN exit runs Gluetun with its instance's settings. Docker is faked: a + // script that records its arguments, behind a unit that does nothing. Like + // Gluetun, its container exits with 1 when stopped. + c.sh(t, `printf '#!/bin/sh\necho "$@" >>/run/docker.args\nif [ "$1" = run ]; then trap "exit 1" TERM; sleep infinity & wait; fi\nexit 0\n' >/usr/bin/docker +chmod +x /usr/bin/docker +printf '[Service]\nType=oneshot\nRemainAfterExit=yes\nExecStart=/bin/true\n' >/run/systemd/system/docker.service +install -m 600 /usr/share/magnetowid/vpn.env.example /etc/magnetowid/vpn-pl.env +systemctl daemon-reload +systemctl enable --now magnetowid-vpn@pl`) + // The example names no VPN provider, so the exit must fail without retrying. + c.await(t, "systemctl is-failed magnetowid-vpn@pl", "failed") + if got := c.sh(t, "systemctl show -p ExecMainStatus -p NRestarts magnetowid-vpn@pl | sort | tr '\\n' ' '"); got != "ExecMainStatus=78 NRestarts=0" { + t.Errorf("the VPN exit without a provider ended with %s", got) + } + if run, _ := c.try("grep '^run ' /run/docker.args"); run != "" { + t.Errorf("the VPN exit ran docker %s without a provider", run) + } + c.sh(t, `sed -i 's/^VPN_SERVICE_PROVIDER=$/VPN_SERVICE_PROVIDER=example/; s/^MAGNETOWID_VPN_PORT=.*/MAGNETOWID_VPN_PORT=8889/' /etc/magnetowid/vpn-pl.env +systemctl restart magnetowid-vpn@pl`) + c.await(t, "systemctl is-active magnetowid-vpn@pl", "active") + run := c.sh(t, "grep '^run ' /run/docker.args") + // Gluetun's proxy stays on its own port, whatever the local one. + for _, want := range []string{"--name magnetowid-vpn-pl ", "--env-file /etc/magnetowid/vpn-pl.env ", + "--publish 127.0.0.1:8889:8888/tcp ", " qmcgaw/gluetun:"} { + if !strings.Contains(run, want) { + t.Errorf("the VPN exit ran docker %s, without %q", run, want) + } + } + // Stopping it is not a failure, and it comes back if it stops by itself. + c.sh(t, "systemctl stop magnetowid-vpn@pl") + if state, _ := c.try("systemctl is-active magnetowid-vpn@pl"); state != "inactive" { + t.Errorf("stopped VPN exit is %s, want inactive", state) + } + c.sh(t, "systemctl start magnetowid-vpn@pl && kill -TERM $(systemctl show -p MainPID --value magnetowid-vpn@pl)") + c.await(t, "systemctl show -p NRestarts --value magnetowid-vpn@pl", "1") + c.await(t, "systemctl is-active magnetowid-vpn@pl", "active") + pid := c.sh(t, "systemctl show -p MainPID --value magnetowid") c.sh(t, d.upgrade) if got := c.sh(t, "systemctl show -p MainPID --value magnetowid"); (got != pid) != d.restarts { t.Errorf("upgrade changed the main PID from %s to %s, want a restart: %v", pid, got, d.restarts) } checkAPIs(ctx, t, base, apiKey, downloadDir) + if state, _ := c.try("systemctl is-active magnetowid-vpn@pl"); state != "active" { + t.Errorf("upgraded package's VPN exit is %s, want active", state) + } c.sh(t, d.remove) - if state, _ := c.try("systemctl is-active magnetowid"); state != "inactive" { - t.Errorf("removed service is %s, want inactive", state) + for _, unit := range []string{"magnetowid", "magnetowid-vpn@pl"} { + if state, _ := c.try("systemctl is-active " + unit); state != "inactive" { + t.Errorf("removed package's %s is %s, want inactive", unit, state) + } + c.sh(t, "test ! -e /etc/systemd/system/multi-user.target.wants/"+unit+".service") } c.sh(t, "test ! -e /usr/lib/systemd/system/magnetowid.service") - c.sh(t, "test ! -e /etc/systemd/system/multi-user.target.wants/magnetowid.service") + c.sh(t, "test ! -e /usr/lib/systemd/system/magnetowid-vpn@.service") + c.sh(t, "test ! -e /usr/share/magnetowid/vpn.env.example") } type container struct { diff --git a/compose.vpn.yaml b/compose.vpn.yaml new file mode 100644 index 0000000..6b157c3 --- /dev/null +++ b/compose.vpn.yaml @@ -0,0 +1,22 @@ +# Adds a VPN exit for a region-locked site to compose.yaml: +# +# docker compose -f compose.yaml -f compose.vpn.yaml up -d +# +# Guide: https://github.com/combor/magnetowid/blob/main/docs/vpn.md +services: + magnetowid: + environment: + # One setting per site, naming the exit in the site's country. + MAGNETOWID_TVP_PROXY: http://vpn-pl:8888 + depends_on: + - vpn-pl + + vpn-pl: + image: qmcgaw/gluetun:v3 + restart: unless-stopped + cap_add: + - NET_ADMIN + devices: + - /dev/net/tun:/dev/net/tun + # Copy vpn.env.example to vpn-pl.env and fill it in. + env_file: vpn-pl.env diff --git a/docs/usage.md b/docs/usage.md index 44bdac8..913384d 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -7,7 +7,9 @@ magnetowid exposes two APIs for Sonarr and Radarr: Supported sites: **TVP VOD** (`tvp`) and **BBC iPlayer** (`bbc`). See the [TVP VOD notes](../internal/provider/tvp/README.md) and the -[BBC iPlayer notes](../internal/provider/bbc/README.md). +[BBC iPlayer notes](../internal/provider/bbc/README.md). Sites limit their +streams to their own country; [Region-locked sites](vpn.md) shows how to use +them from elsewhere. ## Install and run @@ -65,6 +67,9 @@ Group=yourgroup Run `sudo systemctl restart magnetowid` after changing settings. +The packages also install `magnetowid-vpn@.service`, which stays unused until +you set up a VPN exit for a [region-locked site](vpn.md). + ### Build from source Requires Go 1.27.1+ and ffmpeg. @@ -94,6 +99,8 @@ section in `compose.yaml`. For the Linux service, set them in | `-categories` | `MAGNETOWID_CATEGORIES` | `tv,movies` | download categories to offer | | `-ffmpeg` | `MAGNETOWID_FFMPEG` | `ffmpeg` | ffmpeg binary | | `-log-level` | `MAGNETOWID_LOG_LEVEL` | `info` | `debug`, `info`, `warn`, or `error`; debug includes requests and RSS activity | +| `-tvp-proxy` | `MAGNETOWID_TVP_PROXY` | | TVP VOD's HTTP proxy, e.g. `http://127.0.0.1:8888`, or `direct`; see [Region-locked sites](vpn.md) | +| `-bbc-proxy` | `MAGNETOWID_BBC_PROXY` | | BBC iPlayer's HTTP proxy, or `direct`; see [Region-locked sites](vpn.md) | `GET /health` returns `OK` without an API key. The container health check runs `magnetowid -healthcheck`, which queries `MAGNETOWID_LISTEN` and exits with 0 @@ -291,6 +298,7 @@ the queue and history, with the last error of each failed or retrying job. | Downloads finish but are not imported | Mount the shared folder into Sonarr/Radarr, check file permissions and add a Remote Path Mapping if the paths differ. | | A magnetowid release is sent to another download client | Set the indexer's **Download Client** to `magnetowid`. | | Downloads stay queued with `provider unreachable` in the log | Check the network, DNS, and VPN. Jobs resume automatically when the site becomes reachable, without consuming retries. | +| One site finds nothing, or the log shows its results as `unavailable` | The site refuses streams outside its country. See [Region-locked sites](vpn.md). | | A series or film is missing, or its episodes are wrong | Check that the site has it for free, then add an [override](#correcting-matches). | To inspect recent container messages: diff --git a/docs/vpn.md b/docs/vpn.md new file mode 100644 index 0000000..21ecd88 --- /dev/null +++ b/docs/vpn.md @@ -0,0 +1,224 @@ +# Region-locked sites + +Sites limit their streams to their own country: + +| Site | Country | Setting | +|---|---|---| +| TVP VOD | Poland | `MAGNETOWID_TVP_PROXY` | +| BBC iPlayer | United Kingdom | `MAGNETOWID_BBC_PROXY` | + +A site in the country magnetowid runs in needs nothing. For any other site, +run a **VPN exit** in the site's country and name it in the site's setting. +magnetowid then reaches that site, and only that site, through the exit, so +one install can use sites from different countries. + +An exit is a [Gluetun](https://github.com/qdm12/gluetun) container connected +to your VPN provider. You need Docker and a VPN subscription with servers in +the countries you want. + +- [Linux service](#linux-service) +- [Docker Compose](#docker-compose) +- [The site settings](#the-site-settings) +- [Troubleshooting](#troubleshooting) + +## Linux service + +The packages include `magnetowid-vpn@.service`, which runs one exit per name. +The name is yours to choose; these steps use `pl` for an exit in Poland. + +### One site + +**1. Describe the exit.** Copy the example and edit the copy: + +```sh +sudo install -m 600 /usr/share/magnetowid/vpn.env.example /etc/magnetowid/vpn-pl.env +sudoedit /etc/magnetowid/vpn-pl.env +``` + +[The example](../vpn.env.example) looks like this: + +```sh +# The country this exit must be in. +SERVER_COUNTRIES=Poland + +# Your VPN provider and its credentials. Gluetun's wiki lists the lines each +# provider needs: https://github.com/qdm12/gluetun-wiki/tree/main/setup/providers +VPN_SERVICE_PROVIDER= +VPN_TYPE= + +# magnetowid reaches the exit through Gluetun's HTTP proxy. +HTTPPROXY=on + +# Linux service only: the local port of this exit's proxy. Give each exit its own. +MAGNETOWID_VPN_PORT=8888 +``` + +Fill in `VPN_SERVICE_PROVIDER` and `VPN_TYPE`, and add the credential lines +[your provider's page](https://github.com/qdm12/gluetun-wiki/tree/main/setup/providers) +lists. Leave the other lines as they are. + +**2. Start the exit**, now and at every boot: + +```sh +sudo systemctl enable --now magnetowid-vpn@pl +``` + +**3. Check its country.** The first start downloads Gluetun, so allow a minute: + +```sh +curl -x http://127.0.0.1:8888 https://ipinfo.io/country +``` + +This must print `PL`. If it doesn't, see [Troubleshooting](#troubleshooting). + +**4. Send the site through it.** Add to `/etc/magnetowid/magnetowid.env`: + +```sh +MAGNETOWID_TVP_PROXY=http://127.0.0.1:8888 +``` + +```sh +sudo systemctl restart magnetowid +``` + +### A second site + +Repeat the steps with another name, country and port. For BBC iPlayer: + +```sh +sudo install -m 600 /usr/share/magnetowid/vpn.env.example /etc/magnetowid/vpn-gb.env +sudoedit /etc/magnetowid/vpn-gb.env +``` + +In `vpn-gb.env`, fill in the provider lines as before and change two lines: + +```sh +SERVER_COUNTRIES=United Kingdom +MAGNETOWID_VPN_PORT=8889 +``` + +Your provider may want separate credentials, such as a second WireGuard key, +for a second connection at the same time. + +```sh +sudo systemctl enable --now magnetowid-vpn@gb +curl -x http://127.0.0.1:8889 https://ipinfo.io/country +``` + +This must print `GB`. `/etc/magnetowid/magnetowid.env` then names both exits: + +```sh +MAGNETOWID_TVP_PROXY=http://127.0.0.1:8888 +MAGNETOWID_BBC_PROXY=http://127.0.0.1:8889 +``` + +```sh +sudo systemctl restart magnetowid +``` + +## Docker Compose + +[`compose.vpn.yaml`](../compose.vpn.yaml) adds an exit in Poland, `vpn-pl`, to +the [quick start](../README.md#quick-start)'s `compose.yaml`. + +### One site + +**1. Get the files**, in the folder that holds `compose.yaml`: + +```sh +curl -fsSLO https://raw.githubusercontent.com/combor/magnetowid/main/compose.vpn.yaml +curl -fsSL https://raw.githubusercontent.com/combor/magnetowid/main/vpn.env.example -o vpn-pl.env +chmod 600 vpn-pl.env +``` + +**2. Describe the exit.** In `vpn-pl.env`, fill in `VPN_SERVICE_PROVIDER` and +`VPN_TYPE`, and add the credential lines +[your provider's page](https://github.com/qdm12/gluetun-wiki/tree/main/setup/providers) +lists. + +**3. Use both Compose files** from now on. Add to `.env`: + +```sh +COMPOSE_FILE=compose.yaml:compose.vpn.yaml +``` + +**4. Start, and check the exit's country:** + +```sh +docker compose up -d +docker compose exec vpn-pl wget -qO- https://ipinfo.io/country +``` + +This must print `PL`. TVP VOD already goes through the exit: +`compose.vpn.yaml` sets `MAGNETOWID_TVP_PROXY` to `http://vpn-pl:8888`. + +### A second site + +Copy `vpn-pl.env` to `vpn-gb.env`, set `SERVER_COUNTRIES=United Kingdom` in +the copy, and give it separate credentials if your provider wants them for a +second connection. Then add the exit and the site's setting to +`compose.vpn.yaml`: + +```yaml +services: + magnetowid: + environment: + MAGNETOWID_TVP_PROXY: http://vpn-pl:8888 + MAGNETOWID_BBC_PROXY: http://vpn-gb:8888 + depends_on: + - vpn-pl + - vpn-gb + + vpn-pl: + image: qmcgaw/gluetun:v3 + restart: unless-stopped + cap_add: + - NET_ADMIN + devices: + - /dev/net/tun:/dev/net/tun + env_file: vpn-pl.env + + vpn-gb: + image: qmcgaw/gluetun:v3 + restart: unless-stopped + cap_add: + - NET_ADMIN + devices: + - /dev/net/tun:/dev/net/tun + env_file: vpn-gb.env +``` + +```sh +docker compose up -d +docker compose exec vpn-gb wget -qO- https://ipinfo.io/country +``` + +This must print `GB`. + +## The site settings + +Each takes one of: + +| Value | The site is reached | +|---|---| +| `http://host:port` | through that HTTP proxy | +| `direct` | without a proxy, even if `HTTPS_PROXY` is set | +| not set | as before: through `HTTPS_PROXY` if it is set, otherwise directly | + +The command-line flags are `-tvp-proxy` and `-bbc-proxy`. A site's searches, +streams and subtitles all use its setting. magnetowid logs every site's setting +when it starts. + +While an exit is down, its site's downloads wait as `provider unreachable` +and resume by themselves. They are never sent without the exit. + +## Troubleshooting + +| Problem | What to check | +|---|---| +| The exit doesn't start | `systemctl status magnetowid-vpn@pl` and `journalctl -u magnetowid-vpn@pl -n 50`, or `docker compose logs vpn-pl`. A status of `78/CONFIG` means `VPN_SERVICE_PROVIDER` or `MAGNETOWID_VPN_PORT` is missing from the exit's file. Other errors come from Gluetun and name the setting it rejects: a missing credential, or a country your provider's Gluetun page spells differently. | +| Gluetun rejects the country although it is right | `VPN_SERVICE_PROVIDER` is blank, so Gluetun assumed a provider of its own. | +| The country check fails to connect | The exit is still starting, or the port isn't the exit's `MAGNETOWID_VPN_PORT`. | +| One site's results are empty | Its streams are refused: the log's search lines show `unavailable` above zero. Check that site's setting and its exit's country, then restart magnetowid. | +| Downloads wait as `provider unreachable` | The site's exit is down. Start it; the downloads resume. | +| A download fails with an ffmpeg `Input/output error` | magnetowid fetches streams itself where it can; live and encrypted ones are left to ffmpeg, which can't connect through Gluetun's proxy. These fail instead of bypassing the exit. | diff --git a/internal/downloader/ffmpeg.go b/internal/downloader/ffmpeg.go index d375409..973d2a0 100644 --- a/internal/downloader/ffmpeg.go +++ b/internal/downloader/ffmpeg.go @@ -13,6 +13,7 @@ import ( "os" "os/exec" "path/filepath" + "slices" "strconv" "strings" "sync" @@ -33,8 +34,9 @@ const playlistTimeout = 30 * time.Second // Neither transcodes. type FFmpeg struct { Path string // binary; "ffmpeg" if empty - // Fetches playlists and segments; nil uses a default. Each download gets - // its own copy, with a cookie jar if the client has none. + // Fetches playlists and segments, on the stream's Transport if it has one; + // nil uses a default. Each download gets its own copy, with a cookie jar + // if the client has none. Client *http.Client Log *slog.Logger // nil discards // Zero defaults to 5 minutes, allowing for +faststart's silent final pass. @@ -81,6 +83,7 @@ func (f *FFmpeg) Download(ctx context.Context, s provider.Stream, out string, pr c := *f.Client client = &c } + client = s.Client(client) if client.Jar == nil { // ffmpeg sends a playlist's cookies with its segments too. client.Jar, _ = cookiejar.New(nil) @@ -108,13 +111,52 @@ func (f *FFmpeg) Download(ctx context.Context, s provider.Stream, out string, pr if err := os.RemoveAll(dir); err != nil { return err } + env, err := proxyEnv(s) + if err != nil { + return err + } inputs := directInputs(s, m, master) var args []string for _, in := range inputs { args = append(args, headerArgs(s.Header)...) args = append(args, "-i", in) } - return f.run(ctx, "", append(args, outputArgs(len(inputs), m.AudioLanguage, out)...), progress) + return f.run(ctx, "", env, append(args, outputArgs(len(inputs), m.AudioLanguage, out)...), progress) +} + +// proxyEnv returns the environment in which ffmpeg reaches the stream as its +// Transport does, or nil, magnetowid's own, for a stream without one. ffmpeg +// reads http_proxy for HTTP and HTTPS alike. +// +// The proxy for the stream's URL then serves every host ffmpeg fetches from, +// renditions and segments included, so a Transport must use one proxy for all +// hosts. Proxies chosen by host, as the environment's are, need a stream +// without a Transport: ffmpeg applies the environment's rules itself. +func proxyEnv(s provider.Stream) ([]string, error) { + t, ok := s.Transport.(*http.Transport) + if !ok { + return nil, nil + } + env := slices.DeleteFunc(os.Environ(), func(kv string) bool { + k, _, _ := strings.Cut(kv, "=") + return strings.EqualFold(k, "http_proxy") || strings.EqualFold(k, "no_proxy") + }) + if t.Proxy == nil { + return env, nil + } + req, err := http.NewRequest(http.MethodGet, s.URL, nil) + if err != nil { + return nil, err + } + // A proxy that can't be determined must not become a direct connection. + proxy, err := t.Proxy(req) + if err != nil { + return nil, fmt.Errorf("finding the stream's proxy: %w", err) + } + if proxy != nil { + env = append(env, "http_proxy="+proxy.String()) + } + return env, nil } func (f *FFmpeg) log() *slog.Logger { @@ -175,8 +217,9 @@ func (f *FFmpeg) binary() (string, error) { return bin, nil } -// run runs ffmpeg in dir, or the current folder if dir is empty. -func (f *FFmpeg) run(parent context.Context, dir string, args []string, progress func(time.Duration, int64)) error { +// run runs ffmpeg in dir, or the current folder if dir is empty, with env, or +// magnetowid's environment if env is nil. +func (f *FFmpeg) run(parent context.Context, dir string, env, args []string, progress func(time.Duration, int64)) error { bin, err := f.binary() if err != nil { return err @@ -191,6 +234,7 @@ func (f *FFmpeg) run(parent context.Context, dir string, args []string, progress args = append([]string{"-nostdin", "-hide_banner", "-loglevel", "warning", "-y"}, args...) cmd := exec.CommandContext(ctx, bin, args...) cmd.Dir = dir + cmd.Env = env cmd.WaitDelay = 5 * time.Second // Isolate ffmpeg from terminal signals so magnetowid can requeue interrupted jobs. cmd.SysProcAttr = ownProcessGroup() diff --git a/internal/downloader/ffmpeg_unix_test.go b/internal/downloader/ffmpeg_unix_test.go index 3ba6bf8..73e373b 100644 --- a/internal/downloader/ffmpeg_unix_test.go +++ b/internal/downloader/ffmpeg_unix_test.go @@ -8,11 +8,13 @@ import ( "io" "net/http" "net/http/httptest" + "net/url" "os" "path/filepath" "slices" "strconv" "strings" + "sync" "syscall" "testing" "time" @@ -58,11 +60,13 @@ func TestFFmpegRunsInOwnProcessGroup(t *testing.T) { } } -// fakeFFmpeg records its arguments and folder in log and creates its output. +// fakeFFmpeg records its folder, arguments and proxy settings in log and +// creates its output. func fakeFFmpeg(t *testing.T, dir string) (bin, log string) { t.Helper() bin, log = filepath.Join(dir, "ffmpeg"), filepath.Join(dir, "args") - script := "#!/bin/sh\npwd > '" + log + "'\nfor a; do echo \"$a\"; out=$a; done >> '" + log + "'\n: > \"$out\"\n" + script := "#!/bin/sh\npwd > '" + log + "'\nfor a; do echo \"$a\"; out=$a; done >> '" + log + "'\n" + + "echo \"http_proxy=$http_proxy no_proxy=$no_proxy\" >> '" + log + "'\n: > \"$out\"\n" if err := os.WriteFile(bin, []byte(script), 0o755); err != nil { t.Fatal(err) } @@ -110,6 +114,97 @@ func TestFFmpegInputs(t *testing.T) { } } +// ffmpeg reads its proxy from the environment, which must match the stream's +// transport and not magnetowid's own settings. +func TestFFmpegFollowsStreamProxy(t *testing.T) { + dir := t.TempDir() + bin, log := fakeFFmpeg(t, dir) + t.Setenv("http_proxy", "http://environment.example:3128") + t.Setenv("no_proxy", "elsewhere.example") + proxy, err := url.Parse("http://proxy.example:3128") + if err != nil { + t.Fatal(err) + } + for _, tt := range []struct { + name string + transport http.RoundTripper + want string + }{ + {"no transport", nil, "http_proxy=http://environment.example:3128 no_proxy=elsewhere.example"}, + {"proxy", &http.Transport{Proxy: http.ProxyURL(proxy)}, "http_proxy=http://proxy.example:3128 no_proxy="}, + {"direct", &http.Transport{}, "http_proxy= no_proxy="}, + {"no proxy for the stream", &http.Transport{Proxy: func(*http.Request) (*url.URL, error) { return nil, nil }}, + "http_proxy= no_proxy="}, + } { + // Not a playlist, so ffmpeg fetches it. + s := provider.Stream{URL: "http://example.invalid/video.mp4", Transport: tt.transport} + if err := (&FFmpeg{Path: bin}).Download(context.Background(), s, filepath.Join(dir, "out.mp4"), + func(time.Duration, int64) {}); err != nil { + t.Fatalf("%s: %v", tt.name, err) + } + b, err := os.ReadFile(log) + if err != nil { + t.Fatal(err) + } + if lines := strings.Split(strings.TrimSpace(string(b)), "\n"); lines[len(lines)-1] != tt.want { + t.Errorf("%s: ffmpeg ran with %q, want %q", tt.name, lines[len(lines)-1], tt.want) + } + } + + // An unknown proxy must not become a direct connection. + broken := &http.Transport{Proxy: func(*http.Request) (*url.URL, error) { return nil, errors.New("bad proxy") }} + os.Remove(log) + err = (&FFmpeg{Path: bin}).Download(context.Background(), + provider.Stream{URL: "http://example.invalid/video.mp4", Transport: broken}, filepath.Join(dir, "out.mp4"), + func(time.Duration, int64) {}) + if err == nil || !strings.Contains(err.Error(), "bad proxy") { + t.Errorf("err = %v, want the proxy's", err) + } + if _, err := os.Stat(log); !os.IsNotExist(err) { + t.Errorf("ffmpeg ran without knowing its proxy: %v", err) + } +} + +// The stream's transport, e.g. its site's proxy, fetches playlists and segments. +func TestDownloadUsesStreamTransport(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/master.m3u8": + io.WriteString(w, "#EXTM3U\n#EXT-X-STREAM-INF:BANDWIDTH=1\nvod.m3u8\n") + case "/vod.m3u8": + io.WriteString(w, "#EXTM3U\n#EXTINF:4,\n1.ts\n#EXT-X-ENDLIST\n") + default: + io.WriteString(w, "segment data") + } + })) + defer srv.Close() + dir := t.TempDir() + bin, _ := fakeFFmpeg(t, dir) + var mu sync.Mutex + var paths []string + site := roundTripFunc(func(req *http.Request) (*http.Response, error) { + mu.Lock() + paths = append(paths, req.URL.Path) + mu.Unlock() + return http.DefaultTransport.RoundTrip(req) + }) + // The engine's own client must stay unused. + other := &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + t.Errorf("%s fetched without the stream's transport", req.URL.Path) + return nil, errors.New("wrong transport") + })} + err := (&FFmpeg{Path: bin, Client: other}).Download(context.Background(), + provider.Stream{URL: srv.URL + "/master.m3u8", Transport: site}, filepath.Join(dir, "out.mp4"), + func(time.Duration, int64) {}) + if err != nil { + t.Fatal(err) + } + slices.Sort(paths) + if want := []string{"/1.ts", "/master.m3u8", "/vod.m3u8"}; !slices.Equal(paths, want) { + t.Errorf("the stream's transport fetched %q, want %q", paths, want) + } +} + // ffmpeg 5.x reports unreadable inputs without "Error opening input". func TestUnreadableSegmentsRemoved(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/internal/downloader/queue.go b/internal/downloader/queue.go index ea53ed3..709a4ac 100644 --- a/internal/downloader/queue.go +++ b/internal/downloader/queue.go @@ -650,7 +650,7 @@ func (q *Queue) saveSubtitles(ctx context.Context, j Job, s provider.Stream, wor if slices.Contains(paths, path) { continue // more in the same language } - srt, err := subtitles.Fetch(ctx, defaultClient, sub, s.Header) + srt, err := subtitles.Fetch(ctx, s.Client(defaultClient), sub, s.Header) if err == nil { err = os.WriteFile(path, srt, 0o666) } diff --git a/internal/downloader/queue_test.go b/internal/downloader/queue_test.go index ec27732..e9dc1af 100644 --- a/internal/downloader/queue_test.go +++ b/internal/downloader/queue_test.go @@ -15,6 +15,7 @@ import ( "slices" "strings" "sync" + "sync/atomic" "testing" "time" @@ -29,8 +30,13 @@ type fakeProvider struct { offline int err error subtitles []provider.Subtitle + transport http.RoundTripper } +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) } + func (f *fakeProvider) Name() string { return "fake" } func (f *fakeProvider) Search(context.Context, provider.Query) ([]provider.Item, error) { @@ -48,7 +54,8 @@ func (f *fakeProvider) Resolve(_ context.Context, id string) (provider.Stream, e if f.err != nil { return provider.Stream{}, f.err } - return provider.Stream{URL: fmt.Sprintf("http://example.invalid/%s/%d.m3u8", id, f.resolves), Subtitles: f.subtitles}, nil + return provider.Stream{URL: fmt.Sprintf("http://example.invalid/%s/%d.m3u8", id, f.resolves), Subtitles: f.subtitles, + Transport: f.transport}, nil } type fakeEngine struct { @@ -365,7 +372,13 @@ func TestSubtitlesSaved(t *testing.T) {

Dzień dobry.

`) })) defer srv.Close() - p := &fakeProvider{subtitles: []provider.Subtitle{ + // Subtitles come through the stream's transport, e.g. its site's proxy. + var fetched atomic.Int32 + site := roundTripFunc(func(req *http.Request) (*http.Response, error) { + fetched.Add(1) + return http.DefaultTransport.RoundTrip(req) + }) + p := &fakeProvider{transport: site, subtitles: []provider.Subtitle{ {URL: srv.URL + "/napisy.xml", Format: provider.TTML, Language: "pol", SDH: true}, {URL: srv.URL + "/gone.xml", Format: provider.TTML, Language: "ukr"}, }} @@ -389,6 +402,9 @@ func TestSubtitlesSaved(t *testing.T) { if want := "1\n00:00:01,000 --> 00:00:02,500\nDzień dobry.\n\n"; string(srt) != want { t.Errorf("SRT %q, want %q", srt, want) } + if n := fetched.Load(); n != 2 { + t.Errorf("the stream's transport made %d requests, want one for each subtitle", n) + } } func TestSubtitleName(t *testing.T) { diff --git a/internal/downloader/segments.go b/internal/downloader/segments.go index 58d709c..f0dc729 100644 --- a/internal/downloader/segments.go +++ b/internal/downloader/segments.go @@ -93,7 +93,8 @@ func (f *FFmpeg) downloadSegments(ctx context.Context, client *http.Client, h ht if len(tracks) == 2 { args = append(args, "-i", tracks[1].name+".m3u8") } - err = f.run(ctx, dir, append(args, outputArgs(len(tracks), m.AudioLanguage, out)...), func(time.Duration, int64) {}) + // The inputs are local files, so any environment will do. + err = f.run(ctx, dir, nil, append(args, outputArgs(len(tracks), m.AudioLanguage, out)...), func(time.Duration, int64) {}) if errors.Is(err, errDataLoss) || errors.Is(err, errUnreadable) { // A damaged file on disk would fail every retry. if rmErr := os.RemoveAll(dir); rmErr != nil { diff --git a/internal/probe/probe.go b/internal/probe/probe.go index 5620838..4c68d16 100644 --- a/internal/probe/probe.go +++ b/internal/probe/probe.go @@ -204,7 +204,7 @@ func (pr *Prober) probe(ctx context.Context, p provider.Provider, id string) (In if s, err = p.Resolve(ctx, id); err == nil { var m hls.Master var ok bool - if m, ok, err = hls.Load(ctx, client, s); err == nil { + if m, ok, err = hls.Load(ctx, s.Client(client), s); err == nil { if !ok { return Info{}, ErrNotHLS } diff --git a/internal/probe/probe_test.go b/internal/probe/probe_test.go index bec004e..c0a3552 100644 --- a/internal/probe/probe_test.go +++ b/internal/probe/probe_test.go @@ -86,6 +86,7 @@ const master = `#EXTM3U type fakeProvider struct { base string unavailable map[string]bool + transport http.RoundTripper mu sync.Mutex resolves int } @@ -103,9 +104,13 @@ func (f *fakeProvider) Resolve(_ context.Context, id string) (provider.Stream, e if f.unavailable[id] { return provider.Stream{}, fmt.Errorf("%w: DRM-protected", provider.ErrUnavailable) } - return provider.Stream{URL: f.base + "/" + id}, nil + return provider.Stream{URL: f.base + "/" + id, Transport: f.transport}, nil } +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) } + func (f *fakeProvider) count() int { f.mu.Lock() defer f.mu.Unlock() @@ -161,6 +166,22 @@ func TestProbe(t *testing.T) { } } +// The stream's transport, e.g. its site's proxy, fetches the playlist. +func TestProbeUsesStreamTransport(t *testing.T) { + pr, p := newFixture(t, 0) + pr.Client = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + return nil, errors.New("wrong transport") + })} + fetched := 0 + p.transport = roundTripFunc(func(req *http.Request) (*http.Response, error) { + fetched++ + return http.DefaultTransport.RoundTrip(req) + }) + if _, err := pr.Probe(context.Background(), p, "ok.m3u8"); err != nil || fetched != 1 { + t.Errorf("Probe: %v after %d requests on the stream's transport, want 1", err, fetched) + } +} + func TestProbeRetries(t *testing.T) { pr, p := newFixture(t, 2) if got, err := pr.Probe(context.Background(), p, "flaky.m3u8"); err != nil || got.Height != 1080 { diff --git a/internal/provider/bbc/README.md b/internal/provider/bbc/README.md index d2a03b4..58b0b7a 100644 --- a/internal/provider/bbc/README.md +++ b/internal/provider/bbc/README.md @@ -6,7 +6,8 @@ See [Setup and configuration](../../../docs/usage.md) for connection settings. iPlayer streams only to the UK, where watching or downloading BBC programmes requires a TV licence. Elsewhere, searches find programmes but leave every -release out as unavailable. +release out as unavailable; [Region-locked sites](../../../docs/vpn.md) shows +how to give iPlayer a UK connection of its own. ## Series diff --git a/internal/provider/bbc/bbc_test.go b/internal/provider/bbc/bbc_test.go index 233d935..cab2342 100644 --- a/internal/provider/bbc/bbc_test.go +++ b/internal/provider/bbc/bbc_test.go @@ -333,6 +333,10 @@ func TestResolve(t *testing.T) { if !strings.HasSuffix(s.URL, "/cf/vf.ism.hlsv2.ism/iptv_hd_abr_v1_hls_master.m3u8?Expires=1") || s.Header.Get("User-Agent") != userAgent { t.Errorf("stream %+v", s) } + // The stream is fetched the way the API is reached, e.g. through BBC's proxy. + if s.Transport == nil || s.Transport != p.client.Transport { + t.Errorf("stream transport = %v, want the client's", s.Transport) + } // Each CDN's copy, in priority order. srv := strings.TrimSuffix(s.URL, "/cf/vf.ism.hlsv2.ism/iptv_hd_abr_v1_hls_master.m3u8?Expires=1") if want := []provider.Subtitle{ diff --git a/internal/provider/bbc/stream.go b/internal/provider/bbc/stream.go index 6d3d010..b1c71d4 100644 --- a/internal/provider/bbc/stream.go +++ b/internal/provider/bbc/stream.go @@ -181,7 +181,8 @@ func (p *Provider) hlsStream(ctx context.Context, sel selection) (provider.Strea } continue } - s := provider.Stream{URL: final, Header: http.Header{"User-Agent": {userAgent}}, Playlist: string(body)} + s := provider.Stream{URL: final, Header: http.Header{"User-Agent": {userAgent}}, Transport: p.client.Transport, + Playlist: string(body)} if atoi(video.Height) >= 1080 && atoi(video.Bitrate) > 0 { s.Playlist = p.addFullHD(ctx, s, atoi(video.Bitrate)*1000) } diff --git a/internal/provider/provider.go b/internal/provider/provider.go index 93d4673..7d98a57 100644 --- a/internal/provider/provider.go +++ b/internal/provider/provider.go @@ -70,12 +70,25 @@ type Item struct { type Stream struct { URL string // anything ffmpeg can open Header http.Header + // Reaches the site, e.g. through its proxy, to fetch the stream and its + // subtitles; nil leaves the choice to the caller. + Transport http.RoundTripper // The HLS master playlist at URL, if the provider has amended it, e.g. to // add variants the site omits. Its relative URIs resolve against URL. Playlist string Subtitles []Subtitle // saved next to the video } +// Client returns c, on the stream's Transport if it has one. +func (s Stream) Client(c *http.Client) *http.Client { + if s.Transport == nil { + return c + } + site := *c + site.Transport = s.Transport + return &site +} + type Subtitle struct { URL string // fetched with the stream's Header Format string // TTML, the only one read so far diff --git a/internal/provider/tvp/tvp.go b/internal/provider/tvp/tvp.go index b6f4e85..fa4fcd3 100644 --- a/internal/provider/tvp/tvp.go +++ b/internal/provider/tvp/tvp.go @@ -623,7 +623,7 @@ func (p *Provider) Resolve(ctx context.Context, id string) (provider.Stream, err if len(hls) == 0 || hls[0].Src == "" { return provider.Stream{}, fmt.Errorf("%w: no HLS source", provider.ErrUnavailable) } - s := provider.Stream{URL: hls[0].Src, Header: http.Header{"User-Agent": {userAgent}}} + s := provider.Stream{URL: hls[0].Src, Header: http.Header{"User-Agent": {userAgent}}, Transport: p.client.Transport} for _, sub := range pl.Subtitles { if sub.URL != "" { s.Subtitles = append(s.Subtitles, sub.subtitle()) diff --git a/internal/provider/tvp/tvp_test.go b/internal/provider/tvp/tvp_test.go index fbf016a..8b0df70 100644 --- a/internal/provider/tvp/tvp_test.go +++ b/internal/provider/tvp/tvp_test.go @@ -276,6 +276,10 @@ func TestResolve(t *testing.T) { if s.URL != "https://cdn.example/video-fmp4.m3u8" || s.Header.Get("User-Agent") == "" { t.Errorf("stream = %+v", s) } + // The stream is fetched the way the API is reached, e.g. through TVP's proxy. + if s.Transport == nil || s.Transport != p.client.Transport { + t.Errorf("stream transport = %v, want the client's", s.Transport) + } wantSubs := []provider.Subtitle{ {URL: "https://s.tvp.pl/repository/attachment/c/b/c/sdh.xml", Format: provider.TTML, Language: "pol", SDH: true}, {URL: "https://s.tvp.pl/repository/attachment/1/e/4/ua.xml", Format: provider.TTML, Language: "ukr"}, diff --git a/packaging/linux/magnetowid-vpn@.service b/packaging/linux/magnetowid-vpn@.service new file mode 100644 index 0000000..e9905f4 --- /dev/null +++ b/packaging/linux/magnetowid-vpn@.service @@ -0,0 +1,42 @@ +# A VPN exit for region-locked sites, one instance per country: magnetowid-vpn@pl +# reads /etc/magnetowid/vpn-pl.env and serves that exit's HTTP proxy on a local +# port. Requires Docker. Guide: +# https://github.com/combor/magnetowid/blob/main/docs/vpn.md + +[Unit] +Description=VPN exit %i for magnetowid +Documentation=https://github.com/combor/magnetowid/blob/main/docs/vpn.md +Requires=docker.service +After=docker.service network-online.target +Wants=network-online.target +# Ordering only: magnetowid pauses a site's downloads while its exit is down. +Before=magnetowid.service + +[Service] +Type=exec +# Set MAGNETOWID_VPN_IMAGE in the exit's file to run another version. Gluetun +# reads that file too and takes names such as PROXY_PORT for its own settings, +# hence the prefix on this unit's. +Environment=MAGNETOWID_VPN_IMAGE=qmcgaw/gluetun:v3 +EnvironmentFile=/etc/magnetowid/vpn-%i.env +ExecStartPre=-/usr/bin/docker rm --force magnetowid-vpn-%i +ExecStart=/bin/sh -c '\ + if [ -z "$$VPN_SERVICE_PROVIDER" ] || [ -z "$$MAGNETOWID_VPN_PORT" ]; then \ + echo "Set VPN_SERVICE_PROVIDER and MAGNETOWID_VPN_PORT in /etc/magnetowid/vpn-%i.env" >&2; \ + exit 78; \ + fi; \ + exec /usr/bin/docker run --rm --name magnetowid-vpn-%i \ + --cap-add NET_ADMIN --device /dev/net/tun \ + --env-file /etc/magnetowid/vpn-%i.env \ + --publish "127.0.0.1:$$MAGNETOWID_VPN_PORT:8888/tcp" \ + "$$MAGNETOWID_VPN_IMAGE"' +ExecStop=/usr/bin/docker stop magnetowid-vpn-%i +Restart=always +RestartSec=10 +# Gluetun exits with 1 when it is stopped, too. +SuccessExitStatus=1 +# An exit whose file isn't filled in fails once instead of restarting forever. +RestartPreventExitStatus=78 + +[Install] +WantedBy=multi-user.target diff --git a/packaging/linux/preremove.sh b/packaging/linux/preremove.sh index 59384f7..5e153b1 100755 --- a/packaging/linux/preremove.sh +++ b/packaging/linux/preremove.sh @@ -6,6 +6,9 @@ case "$1" in remove | 0) if command -v systemctl >/dev/null 2>&1; then systemctl disable --now magnetowid.service >/dev/null 2>&1 || true + # Disabling the template disables its instances. + systemctl stop 'magnetowid-vpn@*.service' >/dev/null 2>&1 || true + systemctl disable magnetowid-vpn@.service >/dev/null 2>&1 || true fi ;; esac diff --git a/vpn.env.example b/vpn.env.example new file mode 100644 index 0000000..6370f4e --- /dev/null +++ b/vpn.env.example @@ -0,0 +1,17 @@ +# Settings of one VPN exit, a Gluetun container. Guide: +# https://github.com/combor/magnetowid/blob/main/docs/vpn.md +# Keep comments on their own lines: Docker reads this file too. + +# The country this exit must be in. +SERVER_COUNTRIES=Poland + +# Your VPN provider and its credentials. Gluetun's wiki lists the lines each +# provider needs: https://github.com/qdm12/gluetun-wiki/tree/main/setup/providers +VPN_SERVICE_PROVIDER= +VPN_TYPE= + +# magnetowid reaches the exit through Gluetun's HTTP proxy. +HTTPPROXY=on + +# Linux service only: the local port of this exit's proxy. Give each exit its own. +MAGNETOWID_VPN_PORT=8888