From 2c588a4c72c2b2c3e50bf4346ecce6f1b8258bc4 Mon Sep 17 00:00:00 2001 From: Piotr Komborski <163394+combor@users.noreply.github.com> Date: Thu, 1 Oct 2026 17:11:16 +0100 Subject: [PATCH 1/6] Reach each site through its own proxy Sites stream to their own country only, and one process-wide HTTPS_PROXY can exit in a single country. -tvp-proxy and -bbc-proxy (MAGNETOWID_TVP_PROXY, MAGNETOWID_BBC_PROXY) take an HTTP proxy's URL or "direct"; unset, a site follows the environment as before. Each site gets its own client, and its streams carry that client's transport, so playlists, segments, subtitles and probes go the same way as the site's API. ffmpeg, for the streams it fetches itself, is given the proxy that transport would use instead of inheriting http_proxy. --- cmd/magnetowid/main.go | 66 +++++++++++++++-- cmd/magnetowid/main_test.go | 67 ++++++++++++++++- internal/downloader/ffmpeg.go | 49 ++++++++++-- internal/downloader/ffmpeg_unix_test.go | 99 ++++++++++++++++++++++++- internal/downloader/queue.go | 2 +- internal/downloader/queue_test.go | 20 ++++- internal/downloader/segments.go | 3 +- internal/probe/probe.go | 2 +- internal/probe/probe_test.go | 23 +++++- internal/provider/bbc/bbc_test.go | 4 + internal/provider/bbc/stream.go | 3 +- internal/provider/provider.go | 13 ++++ internal/provider/tvp/tvp.go | 2 +- internal/provider/tvp/tvp_test.go | 4 + 14 files changed, 336 insertions(+), 21 deletions(-) diff --git a/cmd/magnetowid/main.go b/cmd/magnetowid/main.go index 6d80baa..f063fcb 100644 --- a/cmd/magnetowid/main.go +++ b/cmd/magnetowid/main.go @@ -50,6 +50,9 @@ func run(log *slog.Logger, level *slog.LevelVar) error { categories := flag.String("categories", envOr("MAGNETOWID_CATEGORIES", "tv,movies"), "comma-separated download categories") ffmpeg := flag.String("ffmpeg", envOr("MAGNETOWID_FFMPEG", "ffmpeg"), "ffmpeg binary") logLevel := flag.String("log-level", envOr("MAGNETOWID_LOG_LEVEL", "info"), "log level: debug, info, warn or error") + const proxyUsage = `'s HTTP proxy, e.g. http://127.0.0.1:8888, or "direct" for none (default: the environment's)` + tvpProxy := flag.String("tvp-proxy", os.Getenv("MAGNETOWID_TVP_PROXY"), "TVP VOD"+proxyUsage) + bbcProxy := flag.String("bbc-proxy", os.Getenv("MAGNETOWID_BBC_PROXY"), "BBC iPlayer"+proxyUsage) healthcheck := flag.Bool("healthcheck", false, "ask the magnetowid at the listen address whether it is healthy, and exit") flag.Parse() @@ -88,12 +91,20 @@ func run(log *slog.Logger, level *slog.LevelVar) error { } defer db.Close() - httpClient := &http.Client{Timeout: 30 * time.Second} - tvpProvider, err := tvp.New(httpClient, log, db) + // Each site streams to its own country, so each has its own connection. + tvpClient, err := siteClient(*tvpProxy) + if err != nil { + return fmt.Errorf("-tvp-proxy: %w", err) + } + bbcClient, err := siteClient(*bbcProxy) + if err != nil { + return fmt.Errorf("-bbc-proxy: %w", err) + } + tvpProvider, err := tvp.New(tvpClient, log, db) if err != nil { return err } - bbcProvider, err := bbc.New(httpClient, log, db) + bbcProvider, err := bbc.New(bbcClient, log, db) if err != nil { return err } @@ -111,7 +122,8 @@ func run(log *slog.Logger, level *slog.LevelVar) error { return err } mux := http.NewServeMux() - prober := &probe.Prober{Client: httpClient} + // Streams bring their site's connection. + prober := &probe.Prober{Client: &http.Client{Timeout: siteTimeout}} mux.Handle("/{provider}/api", &newznab.Handler{Providers: providers, APIKey: *apiKey, Probe: prober, Log: log}) mux.Handle("/api", &sabnzbd.Handler{Queue: queue, APIKey: *apiKey, Categories: cats, Log: log}) (&overrides.Handler{Store: overrideStore, APIKey: *apiKey, Log: log}).Register(mux) @@ -130,7 +142,8 @@ func run(log *slog.Logger, level *slog.LevelVar) error { srv := &http.Server{Addr: *listen, Handler: logRequests(log, mux), ReadHeaderTimeout: 10 * time.Second} errc := make(chan error, 1) go func() { errc <- srv.ListenAndServe() }() - log.Info("magnetowid listening", "version", version, "addr", *listen, "download_dir", dir, "providers", providers.Names()) + log.Info("magnetowid listening", "version", version, "addr", *listen, "download_dir", dir, "providers", providers.Names(), + "tvp_proxy", proxyName(*tvpProxy), "bbc_proxy", proxyName(*bbcProxy)) var serveErr error select { @@ -168,6 +181,49 @@ func logRequests(log *slog.Logger, next http.Handler) http.Handler { }) } +const ( + siteTimeout = 30 * time.Second + // Skips the environment's proxy. + directProxy = "direct" +) + +// siteClient returns the client for a site's API and streams. proxy is an HTTP +// proxy's URL, "direct", or empty for the environment's HTTPS_PROXY and NO_PROXY. +func siteClient(proxy string) (*http.Client, error) { + t := http.DefaultTransport.(*http.Transport).Clone() + // Keep a connection for each of a download's segment workers. + t.MaxIdleConnsPerHost = 4 + switch proxy { + case "": + case directProxy: + t.Proxy = nil + default: + // ffmpeg, which fetches some streams itself, tunnels through nothing else. + u, err := url.Parse(proxy) + if err != nil || u.Scheme != "http" || u.Host == "" { + // The URL may hold a password. + return nil, fmt.Errorf("want an http:// proxy URL or %q", directProxy) + } + t.Proxy = http.ProxyURL(u) + } + return &http.Client{Timeout: siteTimeout, Transport: t}, nil +} + +// proxyName describes a valid proxy setting for the log, without its password. +func proxyName(proxy string) string { + switch proxy { + case "": + return "environment" + case directProxy: + return proxy + } + u, err := url.Parse(proxy) + if err != nil { + return "invalid" + } + return u.Redacted() +} + func checkHealth(listen string) error { u, err := healthURL(listen) if err != nil { diff --git a/cmd/magnetowid/main_test.go b/cmd/magnetowid/main_test.go index 0874408..e9e74a4 100644 --- a/cmd/magnetowid/main_test.go +++ b/cmd/magnetowid/main_test.go @@ -1,6 +1,11 @@ package main -import "testing" +import ( + "net/http" + "reflect" + "strings" + "testing" +) func TestHealthURL(t *testing.T) { for listen, want := range map[string]string{ @@ -20,3 +25,63 @@ func TestHealthURL(t *testing.T) { t.Error("healthURL accepted a listen address without a port") } } + +func TestSiteClient(t *testing.T) { + req, err := http.NewRequest(http.MethodGet, "https://vod.example/playlist.m3u8", nil) + if err != nil { + t.Fatal(err) + } + transport := func(proxy string) *http.Transport { + t.Helper() + c, err := siteClient(proxy) + if err != nil { + t.Fatalf("siteClient(%q): %v", proxy, err) + } + return c.Transport.(*http.Transport) + } + + // The environment's proxy is read once per process, so compare functions. + if got := transport("").Proxy; reflect.ValueOf(got).Pointer() != reflect.ValueOf(http.ProxyFromEnvironment).Pointer() { + t.Error("an unset proxy doesn't follow the environment") + } + if transport("direct").Proxy != nil { + t.Error(`"direct" still uses a proxy`) + } + for proxy, name := range map[string]string{ + "http://proxy.example:3128": "http://proxy.example:3128", + "http://user:secret@proxy.example:3128": "http://user:xxxxx@proxy.example:3128", + } { + u, err := transport(proxy).Proxy(req) + if err != nil || u == nil || u.String() != proxy { + t.Errorf("siteClient(%q) proxies through %v, %v", proxy, u, err) + } + if got := proxyName(proxy); got != name { + t.Errorf("proxyName(%q) = %q, want %q", proxy, got, name) + } + } + if a, b := transport(""), transport(""); a == b { + t.Error("sites share a transport") + } + + // ffmpeg tunnels through HTTP proxies only. + for _, proxy := range []string{ + "socks5://user:secret@proxy.example:1080", + "https://proxy.example:3128", + "proxy.example:3128", + "http://", + "http://[proxy", + "DIRECT", + } { + _, err := siteClient(proxy) + if err == nil { + t.Errorf("siteClient(%q) accepted the proxy", proxy) + } else if strings.Contains(err.Error(), "secret") { + t.Errorf("siteClient(%q) reports the password: %v", proxy, err) + } + } + for proxy, want := range map[string]string{"": "environment", "direct": "direct"} { + if got := proxyName(proxy); got != want { + t.Errorf("proxyName(%q) = %q, want %q", proxy, got, want) + } + } +} diff --git a/internal/downloader/ffmpeg.go b/internal/downloader/ffmpeg.go index d375409..1eefc57 100644 --- a/internal/downloader/ffmpeg.go +++ b/internal/downloader/ffmpeg.go @@ -13,6 +13,7 @@ import ( "os" "os/exec" "path/filepath" + "slices" "strconv" "strings" "sync" @@ -33,8 +34,9 @@ const playlistTimeout = 30 * time.Second // Neither transcodes. type FFmpeg struct { Path string // binary; "ffmpeg" if empty - // Fetches playlists and segments; nil uses a default. Each download gets - // its own copy, with a cookie jar if the client has none. + // Fetches playlists and segments, on the stream's Transport if it has one; + // nil uses a default. Each download gets its own copy, with a cookie jar + // if the client has none. Client *http.Client Log *slog.Logger // nil discards // Zero defaults to 5 minutes, allowing for +faststart's silent final pass. @@ -81,6 +83,7 @@ func (f *FFmpeg) Download(ctx context.Context, s provider.Stream, out string, pr c := *f.Client client = &c } + client = s.Client(client) if client.Jar == nil { // ffmpeg sends a playlist's cookies with its segments too. client.Jar, _ = cookiejar.New(nil) @@ -108,13 +111,47 @@ func (f *FFmpeg) Download(ctx context.Context, s provider.Stream, out string, pr if err := os.RemoveAll(dir); err != nil { return err } + env, err := proxyEnv(s) + if err != nil { + return err + } inputs := directInputs(s, m, master) var args []string for _, in := range inputs { args = append(args, headerArgs(s.Header)...) args = append(args, "-i", in) } - return f.run(ctx, "", append(args, outputArgs(len(inputs), m.AudioLanguage, out)...), progress) + return f.run(ctx, "", env, append(args, outputArgs(len(inputs), m.AudioLanguage, out)...), progress) +} + +// proxyEnv returns the environment in which ffmpeg reaches the stream as its +// Transport does, or nil, magnetowid's own, for a stream without one. ffmpeg +// reads http_proxy for HTTP and HTTPS alike. +func proxyEnv(s provider.Stream) ([]string, error) { + t, ok := s.Transport.(*http.Transport) + if !ok { + return nil, nil + } + env := slices.DeleteFunc(os.Environ(), func(kv string) bool { + k, _, _ := strings.Cut(kv, "=") + return strings.EqualFold(k, "http_proxy") || strings.EqualFold(k, "no_proxy") + }) + if t.Proxy == nil { + return env, nil + } + req, err := http.NewRequest(http.MethodGet, s.URL, nil) + if err != nil { + return nil, err + } + // A proxy that can't be determined must not become a direct connection. + proxy, err := t.Proxy(req) + if err != nil { + return nil, fmt.Errorf("finding the stream's proxy: %w", err) + } + if proxy != nil { + env = append(env, "http_proxy="+proxy.String()) + } + return env, nil } func (f *FFmpeg) log() *slog.Logger { @@ -175,8 +212,9 @@ func (f *FFmpeg) binary() (string, error) { return bin, nil } -// run runs ffmpeg in dir, or the current folder if dir is empty. -func (f *FFmpeg) run(parent context.Context, dir string, args []string, progress func(time.Duration, int64)) error { +// run runs ffmpeg in dir, or the current folder if dir is empty, with env, or +// magnetowid's environment if env is nil. +func (f *FFmpeg) run(parent context.Context, dir string, env, args []string, progress func(time.Duration, int64)) error { bin, err := f.binary() if err != nil { return err @@ -191,6 +229,7 @@ func (f *FFmpeg) run(parent context.Context, dir string, args []string, progress args = append([]string{"-nostdin", "-hide_banner", "-loglevel", "warning", "-y"}, args...) cmd := exec.CommandContext(ctx, bin, args...) cmd.Dir = dir + cmd.Env = env cmd.WaitDelay = 5 * time.Second // Isolate ffmpeg from terminal signals so magnetowid can requeue interrupted jobs. cmd.SysProcAttr = ownProcessGroup() diff --git a/internal/downloader/ffmpeg_unix_test.go b/internal/downloader/ffmpeg_unix_test.go index 3ba6bf8..73e373b 100644 --- a/internal/downloader/ffmpeg_unix_test.go +++ b/internal/downloader/ffmpeg_unix_test.go @@ -8,11 +8,13 @@ import ( "io" "net/http" "net/http/httptest" + "net/url" "os" "path/filepath" "slices" "strconv" "strings" + "sync" "syscall" "testing" "time" @@ -58,11 +60,13 @@ func TestFFmpegRunsInOwnProcessGroup(t *testing.T) { } } -// fakeFFmpeg records its arguments and folder in log and creates its output. +// fakeFFmpeg records its folder, arguments and proxy settings in log and +// creates its output. func fakeFFmpeg(t *testing.T, dir string) (bin, log string) { t.Helper() bin, log = filepath.Join(dir, "ffmpeg"), filepath.Join(dir, "args") - script := "#!/bin/sh\npwd > '" + log + "'\nfor a; do echo \"$a\"; out=$a; done >> '" + log + "'\n: > \"$out\"\n" + script := "#!/bin/sh\npwd > '" + log + "'\nfor a; do echo \"$a\"; out=$a; done >> '" + log + "'\n" + + "echo \"http_proxy=$http_proxy no_proxy=$no_proxy\" >> '" + log + "'\n: > \"$out\"\n" if err := os.WriteFile(bin, []byte(script), 0o755); err != nil { t.Fatal(err) } @@ -110,6 +114,97 @@ func TestFFmpegInputs(t *testing.T) { } } +// ffmpeg reads its proxy from the environment, which must match the stream's +// transport and not magnetowid's own settings. +func TestFFmpegFollowsStreamProxy(t *testing.T) { + dir := t.TempDir() + bin, log := fakeFFmpeg(t, dir) + t.Setenv("http_proxy", "http://environment.example:3128") + t.Setenv("no_proxy", "elsewhere.example") + proxy, err := url.Parse("http://proxy.example:3128") + if err != nil { + t.Fatal(err) + } + for _, tt := range []struct { + name string + transport http.RoundTripper + want string + }{ + {"no transport", nil, "http_proxy=http://environment.example:3128 no_proxy=elsewhere.example"}, + {"proxy", &http.Transport{Proxy: http.ProxyURL(proxy)}, "http_proxy=http://proxy.example:3128 no_proxy="}, + {"direct", &http.Transport{}, "http_proxy= no_proxy="}, + {"no proxy for the stream", &http.Transport{Proxy: func(*http.Request) (*url.URL, error) { return nil, nil }}, + "http_proxy= no_proxy="}, + } { + // Not a playlist, so ffmpeg fetches it. + s := provider.Stream{URL: "http://example.invalid/video.mp4", Transport: tt.transport} + if err := (&FFmpeg{Path: bin}).Download(context.Background(), s, filepath.Join(dir, "out.mp4"), + func(time.Duration, int64) {}); err != nil { + t.Fatalf("%s: %v", tt.name, err) + } + b, err := os.ReadFile(log) + if err != nil { + t.Fatal(err) + } + if lines := strings.Split(strings.TrimSpace(string(b)), "\n"); lines[len(lines)-1] != tt.want { + t.Errorf("%s: ffmpeg ran with %q, want %q", tt.name, lines[len(lines)-1], tt.want) + } + } + + // An unknown proxy must not become a direct connection. + broken := &http.Transport{Proxy: func(*http.Request) (*url.URL, error) { return nil, errors.New("bad proxy") }} + os.Remove(log) + err = (&FFmpeg{Path: bin}).Download(context.Background(), + provider.Stream{URL: "http://example.invalid/video.mp4", Transport: broken}, filepath.Join(dir, "out.mp4"), + func(time.Duration, int64) {}) + if err == nil || !strings.Contains(err.Error(), "bad proxy") { + t.Errorf("err = %v, want the proxy's", err) + } + if _, err := os.Stat(log); !os.IsNotExist(err) { + t.Errorf("ffmpeg ran without knowing its proxy: %v", err) + } +} + +// The stream's transport, e.g. its site's proxy, fetches playlists and segments. +func TestDownloadUsesStreamTransport(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/master.m3u8": + io.WriteString(w, "#EXTM3U\n#EXT-X-STREAM-INF:BANDWIDTH=1\nvod.m3u8\n") + case "/vod.m3u8": + io.WriteString(w, "#EXTM3U\n#EXTINF:4,\n1.ts\n#EXT-X-ENDLIST\n") + default: + io.WriteString(w, "segment data") + } + })) + defer srv.Close() + dir := t.TempDir() + bin, _ := fakeFFmpeg(t, dir) + var mu sync.Mutex + var paths []string + site := roundTripFunc(func(req *http.Request) (*http.Response, error) { + mu.Lock() + paths = append(paths, req.URL.Path) + mu.Unlock() + return http.DefaultTransport.RoundTrip(req) + }) + // The engine's own client must stay unused. + other := &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + t.Errorf("%s fetched without the stream's transport", req.URL.Path) + return nil, errors.New("wrong transport") + })} + err := (&FFmpeg{Path: bin, Client: other}).Download(context.Background(), + provider.Stream{URL: srv.URL + "/master.m3u8", Transport: site}, filepath.Join(dir, "out.mp4"), + func(time.Duration, int64) {}) + if err != nil { + t.Fatal(err) + } + slices.Sort(paths) + if want := []string{"/1.ts", "/master.m3u8", "/vod.m3u8"}; !slices.Equal(paths, want) { + t.Errorf("the stream's transport fetched %q, want %q", paths, want) + } +} + // ffmpeg 5.x reports unreadable inputs without "Error opening input". func TestUnreadableSegmentsRemoved(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/internal/downloader/queue.go b/internal/downloader/queue.go index ea53ed3..709a4ac 100644 --- a/internal/downloader/queue.go +++ b/internal/downloader/queue.go @@ -650,7 +650,7 @@ func (q *Queue) saveSubtitles(ctx context.Context, j Job, s provider.Stream, wor if slices.Contains(paths, path) { continue // more in the same language } - srt, err := subtitles.Fetch(ctx, defaultClient, sub, s.Header) + srt, err := subtitles.Fetch(ctx, s.Client(defaultClient), sub, s.Header) if err == nil { err = os.WriteFile(path, srt, 0o666) } diff --git a/internal/downloader/queue_test.go b/internal/downloader/queue_test.go index ec27732..e9dc1af 100644 --- a/internal/downloader/queue_test.go +++ b/internal/downloader/queue_test.go @@ -15,6 +15,7 @@ import ( "slices" "strings" "sync" + "sync/atomic" "testing" "time" @@ -29,8 +30,13 @@ type fakeProvider struct { offline int err error subtitles []provider.Subtitle + transport http.RoundTripper } +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) } + func (f *fakeProvider) Name() string { return "fake" } func (f *fakeProvider) Search(context.Context, provider.Query) ([]provider.Item, error) { @@ -48,7 +54,8 @@ func (f *fakeProvider) Resolve(_ context.Context, id string) (provider.Stream, e if f.err != nil { return provider.Stream{}, f.err } - return provider.Stream{URL: fmt.Sprintf("http://example.invalid/%s/%d.m3u8", id, f.resolves), Subtitles: f.subtitles}, nil + return provider.Stream{URL: fmt.Sprintf("http://example.invalid/%s/%d.m3u8", id, f.resolves), Subtitles: f.subtitles, + Transport: f.transport}, nil } type fakeEngine struct { @@ -365,7 +372,13 @@ func TestSubtitlesSaved(t *testing.T) {

Dzień dobry.

`) })) defer srv.Close() - p := &fakeProvider{subtitles: []provider.Subtitle{ + // Subtitles come through the stream's transport, e.g. its site's proxy. + var fetched atomic.Int32 + site := roundTripFunc(func(req *http.Request) (*http.Response, error) { + fetched.Add(1) + return http.DefaultTransport.RoundTrip(req) + }) + p := &fakeProvider{transport: site, subtitles: []provider.Subtitle{ {URL: srv.URL + "/napisy.xml", Format: provider.TTML, Language: "pol", SDH: true}, {URL: srv.URL + "/gone.xml", Format: provider.TTML, Language: "ukr"}, }} @@ -389,6 +402,9 @@ func TestSubtitlesSaved(t *testing.T) { if want := "1\n00:00:01,000 --> 00:00:02,500\nDzień dobry.\n\n"; string(srt) != want { t.Errorf("SRT %q, want %q", srt, want) } + if n := fetched.Load(); n != 2 { + t.Errorf("the stream's transport made %d requests, want one for each subtitle", n) + } } func TestSubtitleName(t *testing.T) { diff --git a/internal/downloader/segments.go b/internal/downloader/segments.go index 58d709c..f0dc729 100644 --- a/internal/downloader/segments.go +++ b/internal/downloader/segments.go @@ -93,7 +93,8 @@ func (f *FFmpeg) downloadSegments(ctx context.Context, client *http.Client, h ht if len(tracks) == 2 { args = append(args, "-i", tracks[1].name+".m3u8") } - err = f.run(ctx, dir, append(args, outputArgs(len(tracks), m.AudioLanguage, out)...), func(time.Duration, int64) {}) + // The inputs are local files, so any environment will do. + err = f.run(ctx, dir, nil, append(args, outputArgs(len(tracks), m.AudioLanguage, out)...), func(time.Duration, int64) {}) if errors.Is(err, errDataLoss) || errors.Is(err, errUnreadable) { // A damaged file on disk would fail every retry. if rmErr := os.RemoveAll(dir); rmErr != nil { diff --git a/internal/probe/probe.go b/internal/probe/probe.go index 5620838..4c68d16 100644 --- a/internal/probe/probe.go +++ b/internal/probe/probe.go @@ -204,7 +204,7 @@ func (pr *Prober) probe(ctx context.Context, p provider.Provider, id string) (In if s, err = p.Resolve(ctx, id); err == nil { var m hls.Master var ok bool - if m, ok, err = hls.Load(ctx, client, s); err == nil { + if m, ok, err = hls.Load(ctx, s.Client(client), s); err == nil { if !ok { return Info{}, ErrNotHLS } diff --git a/internal/probe/probe_test.go b/internal/probe/probe_test.go index bec004e..c0a3552 100644 --- a/internal/probe/probe_test.go +++ b/internal/probe/probe_test.go @@ -86,6 +86,7 @@ const master = `#EXTM3U type fakeProvider struct { base string unavailable map[string]bool + transport http.RoundTripper mu sync.Mutex resolves int } @@ -103,9 +104,13 @@ func (f *fakeProvider) Resolve(_ context.Context, id string) (provider.Stream, e if f.unavailable[id] { return provider.Stream{}, fmt.Errorf("%w: DRM-protected", provider.ErrUnavailable) } - return provider.Stream{URL: f.base + "/" + id}, nil + return provider.Stream{URL: f.base + "/" + id, Transport: f.transport}, nil } +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) } + func (f *fakeProvider) count() int { f.mu.Lock() defer f.mu.Unlock() @@ -161,6 +166,22 @@ func TestProbe(t *testing.T) { } } +// The stream's transport, e.g. its site's proxy, fetches the playlist. +func TestProbeUsesStreamTransport(t *testing.T) { + pr, p := newFixture(t, 0) + pr.Client = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + return nil, errors.New("wrong transport") + })} + fetched := 0 + p.transport = roundTripFunc(func(req *http.Request) (*http.Response, error) { + fetched++ + return http.DefaultTransport.RoundTrip(req) + }) + if _, err := pr.Probe(context.Background(), p, "ok.m3u8"); err != nil || fetched != 1 { + t.Errorf("Probe: %v after %d requests on the stream's transport, want 1", err, fetched) + } +} + func TestProbeRetries(t *testing.T) { pr, p := newFixture(t, 2) if got, err := pr.Probe(context.Background(), p, "flaky.m3u8"); err != nil || got.Height != 1080 { diff --git a/internal/provider/bbc/bbc_test.go b/internal/provider/bbc/bbc_test.go index 233d935..cab2342 100644 --- a/internal/provider/bbc/bbc_test.go +++ b/internal/provider/bbc/bbc_test.go @@ -333,6 +333,10 @@ func TestResolve(t *testing.T) { if !strings.HasSuffix(s.URL, "/cf/vf.ism.hlsv2.ism/iptv_hd_abr_v1_hls_master.m3u8?Expires=1") || s.Header.Get("User-Agent") != userAgent { t.Errorf("stream %+v", s) } + // The stream is fetched the way the API is reached, e.g. through BBC's proxy. + if s.Transport == nil || s.Transport != p.client.Transport { + t.Errorf("stream transport = %v, want the client's", s.Transport) + } // Each CDN's copy, in priority order. srv := strings.TrimSuffix(s.URL, "/cf/vf.ism.hlsv2.ism/iptv_hd_abr_v1_hls_master.m3u8?Expires=1") if want := []provider.Subtitle{ diff --git a/internal/provider/bbc/stream.go b/internal/provider/bbc/stream.go index 6d3d010..b1c71d4 100644 --- a/internal/provider/bbc/stream.go +++ b/internal/provider/bbc/stream.go @@ -181,7 +181,8 @@ func (p *Provider) hlsStream(ctx context.Context, sel selection) (provider.Strea } continue } - s := provider.Stream{URL: final, Header: http.Header{"User-Agent": {userAgent}}, Playlist: string(body)} + s := provider.Stream{URL: final, Header: http.Header{"User-Agent": {userAgent}}, Transport: p.client.Transport, + Playlist: string(body)} if atoi(video.Height) >= 1080 && atoi(video.Bitrate) > 0 { s.Playlist = p.addFullHD(ctx, s, atoi(video.Bitrate)*1000) } diff --git a/internal/provider/provider.go b/internal/provider/provider.go index 93d4673..7d98a57 100644 --- a/internal/provider/provider.go +++ b/internal/provider/provider.go @@ -70,12 +70,25 @@ type Item struct { type Stream struct { URL string // anything ffmpeg can open Header http.Header + // Reaches the site, e.g. through its proxy, to fetch the stream and its + // subtitles; nil leaves the choice to the caller. + Transport http.RoundTripper // The HLS master playlist at URL, if the provider has amended it, e.g. to // add variants the site omits. Its relative URIs resolve against URL. Playlist string Subtitles []Subtitle // saved next to the video } +// Client returns c, on the stream's Transport if it has one. +func (s Stream) Client(c *http.Client) *http.Client { + if s.Transport == nil { + return c + } + site := *c + site.Transport = s.Transport + return &site +} + type Subtitle struct { URL string // fetched with the stream's Header Format string // TTML, the only one read so far diff --git a/internal/provider/tvp/tvp.go b/internal/provider/tvp/tvp.go index b6f4e85..fa4fcd3 100644 --- a/internal/provider/tvp/tvp.go +++ b/internal/provider/tvp/tvp.go @@ -623,7 +623,7 @@ func (p *Provider) Resolve(ctx context.Context, id string) (provider.Stream, err if len(hls) == 0 || hls[0].Src == "" { return provider.Stream{}, fmt.Errorf("%w: no HLS source", provider.ErrUnavailable) } - s := provider.Stream{URL: hls[0].Src, Header: http.Header{"User-Agent": {userAgent}}} + s := provider.Stream{URL: hls[0].Src, Header: http.Header{"User-Agent": {userAgent}}, Transport: p.client.Transport} for _, sub := range pl.Subtitles { if sub.URL != "" { s.Subtitles = append(s.Subtitles, sub.subtitle()) diff --git a/internal/provider/tvp/tvp_test.go b/internal/provider/tvp/tvp_test.go index fbf016a..8b0df70 100644 --- a/internal/provider/tvp/tvp_test.go +++ b/internal/provider/tvp/tvp_test.go @@ -276,6 +276,10 @@ func TestResolve(t *testing.T) { if s.URL != "https://cdn.example/video-fmp4.m3u8" || s.Header.Get("User-Agent") == "" { t.Errorf("stream = %+v", s) } + // The stream is fetched the way the API is reached, e.g. through TVP's proxy. + if s.Transport == nil || s.Transport != p.client.Transport { + t.Errorf("stream transport = %v, want the client's", s.Transport) + } wantSubs := []provider.Subtitle{ {URL: "https://s.tvp.pl/repository/attachment/c/b/c/sdh.xml", Format: provider.TTML, Language: "pol", SDH: true}, {URL: "https://s.tvp.pl/repository/attachment/1/e/4/ua.xml", Format: provider.TTML, Language: "ukr"}, From 7c401d9607d4b945490a696d884667b640986298 Mon Sep 17 00:00:00 2001 From: Piotr Komborski <163394+combor@users.noreply.github.com> Date: Thu, 1 Oct 2026 17:11:16 +0100 Subject: [PATCH 2/6] Package a systemd template for VPN exits magnetowid-vpn@ runs a Gluetun container from /etc/magnetowid/vpn-.env and serves its HTTP proxy on a local port, one instance per country. Nothing starts until an instance is enabled, and Docker stays optional. An exit whose file names no provider fails once with status 78 instead of restarting forever; the check sits in the main command because RestartPreventExitStatus ignores ExecStartPre. Removing the package stops and disables the instances. The package smoke test starts an exit against a fake docker on Debian, Fedora and Arch. --- .goreleaser.yml | 17 ++++++++++ cmd/magnetowid/package_smoke_test.go | 43 ++++++++++++++++++++++--- packaging/linux/magnetowid-vpn@.service | 40 +++++++++++++++++++++++ packaging/linux/preremove.sh | 3 ++ vpn.env.example | 17 ++++++++++ 5 files changed, 116 insertions(+), 4 deletions(-) create mode 100644 packaging/linux/magnetowid-vpn@.service create mode 100644 vpn.env.example diff --git a/.goreleaser.yml b/.goreleaser.yml index ada352f..7316c4d 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -63,6 +63,12 @@ archives: - src: packaging/linux/magnetowid.{service,sysusers,env} dst: systemd strip_parent: true + - src: packaging/linux/magnetowid-vpn@.service + dst: systemd + strip_parent: true + - src: vpn.env.example + dst: systemd + strip_parent: true checksum: name_template: '{{ .ProjectName }}_{{ .Version }}_SHA256SUMS' @@ -106,6 +112,13 @@ nfpms: dst: /usr/lib/systemd/system/magnetowid.service - src: packaging/linux/magnetowid.sysusers dst: /usr/lib/sysusers.d/magnetowid.conf + - src: packaging/linux/magnetowid-vpn@.service + dst: /usr/lib/systemd/system/magnetowid-vpn@.service + # Outside /usr/share/doc, which minimal installs leave out. + - dst: /usr/share/magnetowid + type: dir + - src: vpn.env.example + dst: /usr/share/magnetowid/vpn.env.example - dst: /etc/magnetowid type: dir - src: packaging/linux/magnetowid.env @@ -135,6 +148,8 @@ aurs: depends: - ca-certificates - ffmpeg + optdepends: + - 'docker: VPN exits for region-locked sites' backup: - etc/magnetowid/magnetowid.env install: packaging/linux/magnetowid.install @@ -142,7 +157,9 @@ aurs: install -Dm755 "./magnetowid" "${pkgdir}/usr/bin/magnetowid" install -Dm644 "./LICENSE" "${pkgdir}/usr/share/licenses/magnetowid-bin/LICENSE" install -Dm644 "./systemd/magnetowid.service" "${pkgdir}/usr/lib/systemd/system/magnetowid.service" + install -Dm644 "./systemd/magnetowid-vpn@.service" "${pkgdir}/usr/lib/systemd/system/magnetowid-vpn@.service" install -Dm644 "./systemd/magnetowid.sysusers" "${pkgdir}/usr/lib/sysusers.d/magnetowid.conf" + install -Dm644 "./systemd/vpn.env.example" "${pkgdir}/usr/share/magnetowid/vpn.env.example" install -Dm600 "./systemd/magnetowid.env" "${pkgdir}/etc/magnetowid/magnetowid.env" homebrew_casks: diff --git a/cmd/magnetowid/package_smoke_test.go b/cmd/magnetowid/package_smoke_test.go index a264f50..212512c 100644 --- a/cmd/magnetowid/package_smoke_test.go +++ b/cmd/magnetowid/package_smoke_test.go @@ -61,7 +61,7 @@ chown builder ./* runuser -u builder -- makepkg --nodeps pacman -U --noconfirm magnetowid-bin-*.pkg.tar.zst`, upgrade: "pacman -U --noconfirm /build/magnetowid-bin-*.pkg.tar.zst", - remove: "systemctl disable --now magnetowid && pacman -R --noconfirm magnetowid-bin", + remove: "systemctl disable --now magnetowid magnetowid-vpn@pl && pacman -R --noconfirm magnetowid-bin", x264: true, }, } @@ -166,19 +166,54 @@ systemctl start magnetowid`) t.Errorf("category folder is %s, want 775 magnetowid:media", got) } + // A VPN exit runs Gluetun with its instance's settings. Docker is faked: a + // script that records its arguments, behind a unit that does nothing. + c.sh(t, `printf '#!/bin/sh\necho "$@" >>/run/docker.args\n[ "$1" = run ] && exec sleep infinity\nexit 0\n' >/usr/bin/docker +chmod +x /usr/bin/docker +printf '[Service]\nType=oneshot\nRemainAfterExit=yes\nExecStart=/bin/true\n' >/run/systemd/system/docker.service +install -m 600 /usr/share/magnetowid/vpn.env.example /etc/magnetowid/vpn-pl.env +systemctl daemon-reload +systemctl enable --now magnetowid-vpn@pl`) + // The example names no VPN provider, so the exit must fail without retrying. + c.await(t, "systemctl is-failed magnetowid-vpn@pl", "failed") + if got := c.sh(t, "systemctl show -p ExecMainStatus -p NRestarts magnetowid-vpn@pl | sort | tr '\\n' ' '"); got != "ExecMainStatus=78 NRestarts=0" { + t.Errorf("the VPN exit without a provider ended with %s", got) + } + if run, _ := c.try("grep '^run ' /run/docker.args"); run != "" { + t.Errorf("the VPN exit ran docker %s without a provider", run) + } + c.sh(t, `sed -i 's/^VPN_SERVICE_PROVIDER=$/VPN_SERVICE_PROVIDER=example/; s/^MAGNETOWID_VPN_PORT=.*/MAGNETOWID_VPN_PORT=8889/' /etc/magnetowid/vpn-pl.env +systemctl restart magnetowid-vpn@pl`) + c.await(t, "systemctl is-active magnetowid-vpn@pl", "active") + run := c.sh(t, "grep '^run ' /run/docker.args") + // Gluetun's proxy stays on its own port, whatever the local one. + for _, want := range []string{"--name magnetowid-vpn-pl ", "--env-file /etc/magnetowid/vpn-pl.env ", + "--publish 127.0.0.1:8889:8888/tcp ", " qmcgaw/gluetun:"} { + if !strings.Contains(run, want) { + t.Errorf("the VPN exit ran docker %s, without %q", run, want) + } + } + pid := c.sh(t, "systemctl show -p MainPID --value magnetowid") c.sh(t, d.upgrade) if got := c.sh(t, "systemctl show -p MainPID --value magnetowid"); (got != pid) != d.restarts { t.Errorf("upgrade changed the main PID from %s to %s, want a restart: %v", pid, got, d.restarts) } checkAPIs(ctx, t, base, apiKey, downloadDir) + if state, _ := c.try("systemctl is-active magnetowid-vpn@pl"); state != "active" { + t.Errorf("upgraded package's VPN exit is %s, want active", state) + } c.sh(t, d.remove) - if state, _ := c.try("systemctl is-active magnetowid"); state != "inactive" { - t.Errorf("removed service is %s, want inactive", state) + for _, unit := range []string{"magnetowid", "magnetowid-vpn@pl"} { + if state, _ := c.try("systemctl is-active " + unit); state != "inactive" { + t.Errorf("removed package's %s is %s, want inactive", unit, state) + } + c.sh(t, "test ! -e /etc/systemd/system/multi-user.target.wants/"+unit+".service") } c.sh(t, "test ! -e /usr/lib/systemd/system/magnetowid.service") - c.sh(t, "test ! -e /etc/systemd/system/multi-user.target.wants/magnetowid.service") + c.sh(t, "test ! -e /usr/lib/systemd/system/magnetowid-vpn@.service") + c.sh(t, "test ! -e /usr/share/magnetowid/vpn.env.example") } type container struct { diff --git a/packaging/linux/magnetowid-vpn@.service b/packaging/linux/magnetowid-vpn@.service new file mode 100644 index 0000000..847171d --- /dev/null +++ b/packaging/linux/magnetowid-vpn@.service @@ -0,0 +1,40 @@ +# A VPN exit for region-locked sites, one instance per country: magnetowid-vpn@pl +# reads /etc/magnetowid/vpn-pl.env and serves that exit's HTTP proxy on a local +# port. Requires Docker. Guide: +# https://github.com/combor/magnetowid/blob/main/docs/vpn.md + +[Unit] +Description=VPN exit %i for magnetowid +Documentation=https://github.com/combor/magnetowid/blob/main/docs/vpn.md +Requires=docker.service +After=docker.service network-online.target +Wants=network-online.target +# Ordering only: magnetowid pauses a site's downloads while its exit is down. +Before=magnetowid.service + +[Service] +Type=exec +# Set MAGNETOWID_VPN_IMAGE in the exit's file to run another version. Gluetun +# reads that file too and takes names such as PROXY_PORT for its own settings, +# hence the prefix on this unit's. +Environment=MAGNETOWID_VPN_IMAGE=qmcgaw/gluetun:v3 +EnvironmentFile=/etc/magnetowid/vpn-%i.env +ExecStartPre=-/usr/bin/docker rm --force magnetowid-vpn-%i +ExecStart=/bin/sh -c '\ + if [ -z "$$VPN_SERVICE_PROVIDER" ] || [ -z "$$MAGNETOWID_VPN_PORT" ]; then \ + echo "Set VPN_SERVICE_PROVIDER and MAGNETOWID_VPN_PORT in /etc/magnetowid/vpn-%i.env" >&2; \ + exit 78; \ + fi; \ + exec /usr/bin/docker run --rm --name magnetowid-vpn-%i \ + --cap-add NET_ADMIN --device /dev/net/tun \ + --env-file /etc/magnetowid/vpn-%i.env \ + --publish "127.0.0.1:$$MAGNETOWID_VPN_PORT:8888/tcp" \ + "$$MAGNETOWID_VPN_IMAGE"' +ExecStop=/usr/bin/docker stop magnetowid-vpn-%i +Restart=on-failure +RestartSec=10 +# An exit whose file isn't filled in fails once instead of restarting forever. +RestartPreventExitStatus=78 + +[Install] +WantedBy=multi-user.target diff --git a/packaging/linux/preremove.sh b/packaging/linux/preremove.sh index 59384f7..5e153b1 100755 --- a/packaging/linux/preremove.sh +++ b/packaging/linux/preremove.sh @@ -6,6 +6,9 @@ case "$1" in remove | 0) if command -v systemctl >/dev/null 2>&1; then systemctl disable --now magnetowid.service >/dev/null 2>&1 || true + # Disabling the template disables its instances. + systemctl stop 'magnetowid-vpn@*.service' >/dev/null 2>&1 || true + systemctl disable magnetowid-vpn@.service >/dev/null 2>&1 || true fi ;; esac diff --git a/vpn.env.example b/vpn.env.example new file mode 100644 index 0000000..6370f4e --- /dev/null +++ b/vpn.env.example @@ -0,0 +1,17 @@ +# Settings of one VPN exit, a Gluetun container. Guide: +# https://github.com/combor/magnetowid/blob/main/docs/vpn.md +# Keep comments on their own lines: Docker reads this file too. + +# The country this exit must be in. +SERVER_COUNTRIES=Poland + +# Your VPN provider and its credentials. Gluetun's wiki lists the lines each +# provider needs: https://github.com/qdm12/gluetun-wiki/tree/main/setup/providers +VPN_SERVICE_PROVIDER= +VPN_TYPE= + +# magnetowid reaches the exit through Gluetun's HTTP proxy. +HTTPPROXY=on + +# Linux service only: the local port of this exit's proxy. Give each exit its own. +MAGNETOWID_VPN_PORT=8888 From ecbcb1fd58119d4a9ea6013f80cd57dbe3d89ed5 Mon Sep 17 00:00:00 2001 From: Piotr Komborski <163394+combor@users.noreply.github.com> Date: Thu, 1 Oct 2026 17:11:16 +0100 Subject: [PATCH 3/6] Document VPN exits for region-locked sites docs/vpn.md walks through one exit for one site and then a second, for the Linux service and for Docker Compose, with compose.vpn.yaml as the Compose example. --- README.md | 6 +- compose.vpn.yaml | 22 ++++ docs/usage.md | 10 +- docs/vpn.md | 224 ++++++++++++++++++++++++++++++++ internal/provider/bbc/README.md | 3 +- 5 files changed, 261 insertions(+), 4 deletions(-) create mode 100644 compose.vpn.yaml create mode 100644 docs/vpn.md diff --git a/README.md b/README.md index 9cb580c..4acedea 100644 --- a/README.md +++ b/README.md @@ -44,11 +44,12 @@ itself. No separate SABnzbd installation is needed. | Site | Indexer | Good to know | |---|---|---| -| **TVP VOD** | `/tvp` | Polish series and films. [TVP VOD notes](internal/provider/tvp/README.md) | +| **TVP VOD** | `/tvp` | Polish series and films. Streams may need a Polish connection. [TVP VOD notes](internal/provider/tvp/README.md) | | **BBC iPlayer** | `/bbc` | Streams need a UK connection. [BBC iPlayer notes](internal/provider/bbc/README.md) | Only what a site offers for free is found: DRM-protected, paid and -region-blocked streams are left out of the results. +region-blocked streams are left out of the results. To use a site from another +country, give it a VPN exit: see [Region-locked sites](docs/vpn.md). ## What you get @@ -156,6 +157,7 @@ See [Web interface](docs/usage.md#web-interface) for the details. ## Documentation - [Setup and configuration](docs/usage.md): installation options and every setting. +- [Region-locked sites](docs/vpn.md): a VPN exit for each site's country. - [Correcting matches](docs/usage.md#correcting-matches): overrides for titles, seasons and single episodes. - [Troubleshooting](docs/usage.md#troubleshooting): help with connections, downloads and imports. - [Limitations](docs/usage.md#limitations): what magnetowid can't do. diff --git a/compose.vpn.yaml b/compose.vpn.yaml new file mode 100644 index 0000000..6b157c3 --- /dev/null +++ b/compose.vpn.yaml @@ -0,0 +1,22 @@ +# Adds a VPN exit for a region-locked site to compose.yaml: +# +# docker compose -f compose.yaml -f compose.vpn.yaml up -d +# +# Guide: https://github.com/combor/magnetowid/blob/main/docs/vpn.md +services: + magnetowid: + environment: + # One setting per site, naming the exit in the site's country. + MAGNETOWID_TVP_PROXY: http://vpn-pl:8888 + depends_on: + - vpn-pl + + vpn-pl: + image: qmcgaw/gluetun:v3 + restart: unless-stopped + cap_add: + - NET_ADMIN + devices: + - /dev/net/tun:/dev/net/tun + # Copy vpn.env.example to vpn-pl.env and fill it in. + env_file: vpn-pl.env diff --git a/docs/usage.md b/docs/usage.md index 44bdac8..913384d 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -7,7 +7,9 @@ magnetowid exposes two APIs for Sonarr and Radarr: Supported sites: **TVP VOD** (`tvp`) and **BBC iPlayer** (`bbc`). See the [TVP VOD notes](../internal/provider/tvp/README.md) and the -[BBC iPlayer notes](../internal/provider/bbc/README.md). +[BBC iPlayer notes](../internal/provider/bbc/README.md). Sites limit their +streams to their own country; [Region-locked sites](vpn.md) shows how to use +them from elsewhere. ## Install and run @@ -65,6 +67,9 @@ Group=yourgroup Run `sudo systemctl restart magnetowid` after changing settings. +The packages also install `magnetowid-vpn@.service`, which stays unused until +you set up a VPN exit for a [region-locked site](vpn.md). + ### Build from source Requires Go 1.27.1+ and ffmpeg. @@ -94,6 +99,8 @@ section in `compose.yaml`. For the Linux service, set them in | `-categories` | `MAGNETOWID_CATEGORIES` | `tv,movies` | download categories to offer | | `-ffmpeg` | `MAGNETOWID_FFMPEG` | `ffmpeg` | ffmpeg binary | | `-log-level` | `MAGNETOWID_LOG_LEVEL` | `info` | `debug`, `info`, `warn`, or `error`; debug includes requests and RSS activity | +| `-tvp-proxy` | `MAGNETOWID_TVP_PROXY` | | TVP VOD's HTTP proxy, e.g. `http://127.0.0.1:8888`, or `direct`; see [Region-locked sites](vpn.md) | +| `-bbc-proxy` | `MAGNETOWID_BBC_PROXY` | | BBC iPlayer's HTTP proxy, or `direct`; see [Region-locked sites](vpn.md) | `GET /health` returns `OK` without an API key. The container health check runs `magnetowid -healthcheck`, which queries `MAGNETOWID_LISTEN` and exits with 0 @@ -291,6 +298,7 @@ the queue and history, with the last error of each failed or retrying job. | Downloads finish but are not imported | Mount the shared folder into Sonarr/Radarr, check file permissions and add a Remote Path Mapping if the paths differ. | | A magnetowid release is sent to another download client | Set the indexer's **Download Client** to `magnetowid`. | | Downloads stay queued with `provider unreachable` in the log | Check the network, DNS, and VPN. Jobs resume automatically when the site becomes reachable, without consuming retries. | +| One site finds nothing, or the log shows its results as `unavailable` | The site refuses streams outside its country. See [Region-locked sites](vpn.md). | | A series or film is missing, or its episodes are wrong | Check that the site has it for free, then add an [override](#correcting-matches). | To inspect recent container messages: diff --git a/docs/vpn.md b/docs/vpn.md new file mode 100644 index 0000000..21ecd88 --- /dev/null +++ b/docs/vpn.md @@ -0,0 +1,224 @@ +# Region-locked sites + +Sites limit their streams to their own country: + +| Site | Country | Setting | +|---|---|---| +| TVP VOD | Poland | `MAGNETOWID_TVP_PROXY` | +| BBC iPlayer | United Kingdom | `MAGNETOWID_BBC_PROXY` | + +A site in the country magnetowid runs in needs nothing. For any other site, +run a **VPN exit** in the site's country and name it in the site's setting. +magnetowid then reaches that site, and only that site, through the exit, so +one install can use sites from different countries. + +An exit is a [Gluetun](https://github.com/qdm12/gluetun) container connected +to your VPN provider. You need Docker and a VPN subscription with servers in +the countries you want. + +- [Linux service](#linux-service) +- [Docker Compose](#docker-compose) +- [The site settings](#the-site-settings) +- [Troubleshooting](#troubleshooting) + +## Linux service + +The packages include `magnetowid-vpn@.service`, which runs one exit per name. +The name is yours to choose; these steps use `pl` for an exit in Poland. + +### One site + +**1. Describe the exit.** Copy the example and edit the copy: + +```sh +sudo install -m 600 /usr/share/magnetowid/vpn.env.example /etc/magnetowid/vpn-pl.env +sudoedit /etc/magnetowid/vpn-pl.env +``` + +[The example](../vpn.env.example) looks like this: + +```sh +# The country this exit must be in. +SERVER_COUNTRIES=Poland + +# Your VPN provider and its credentials. Gluetun's wiki lists the lines each +# provider needs: https://github.com/qdm12/gluetun-wiki/tree/main/setup/providers +VPN_SERVICE_PROVIDER= +VPN_TYPE= + +# magnetowid reaches the exit through Gluetun's HTTP proxy. +HTTPPROXY=on + +# Linux service only: the local port of this exit's proxy. Give each exit its own. +MAGNETOWID_VPN_PORT=8888 +``` + +Fill in `VPN_SERVICE_PROVIDER` and `VPN_TYPE`, and add the credential lines +[your provider's page](https://github.com/qdm12/gluetun-wiki/tree/main/setup/providers) +lists. Leave the other lines as they are. + +**2. Start the exit**, now and at every boot: + +```sh +sudo systemctl enable --now magnetowid-vpn@pl +``` + +**3. Check its country.** The first start downloads Gluetun, so allow a minute: + +```sh +curl -x http://127.0.0.1:8888 https://ipinfo.io/country +``` + +This must print `PL`. If it doesn't, see [Troubleshooting](#troubleshooting). + +**4. Send the site through it.** Add to `/etc/magnetowid/magnetowid.env`: + +```sh +MAGNETOWID_TVP_PROXY=http://127.0.0.1:8888 +``` + +```sh +sudo systemctl restart magnetowid +``` + +### A second site + +Repeat the steps with another name, country and port. For BBC iPlayer: + +```sh +sudo install -m 600 /usr/share/magnetowid/vpn.env.example /etc/magnetowid/vpn-gb.env +sudoedit /etc/magnetowid/vpn-gb.env +``` + +In `vpn-gb.env`, fill in the provider lines as before and change two lines: + +```sh +SERVER_COUNTRIES=United Kingdom +MAGNETOWID_VPN_PORT=8889 +``` + +Your provider may want separate credentials, such as a second WireGuard key, +for a second connection at the same time. + +```sh +sudo systemctl enable --now magnetowid-vpn@gb +curl -x http://127.0.0.1:8889 https://ipinfo.io/country +``` + +This must print `GB`. `/etc/magnetowid/magnetowid.env` then names both exits: + +```sh +MAGNETOWID_TVP_PROXY=http://127.0.0.1:8888 +MAGNETOWID_BBC_PROXY=http://127.0.0.1:8889 +``` + +```sh +sudo systemctl restart magnetowid +``` + +## Docker Compose + +[`compose.vpn.yaml`](../compose.vpn.yaml) adds an exit in Poland, `vpn-pl`, to +the [quick start](../README.md#quick-start)'s `compose.yaml`. + +### One site + +**1. Get the files**, in the folder that holds `compose.yaml`: + +```sh +curl -fsSLO https://raw.githubusercontent.com/combor/magnetowid/main/compose.vpn.yaml +curl -fsSL https://raw.githubusercontent.com/combor/magnetowid/main/vpn.env.example -o vpn-pl.env +chmod 600 vpn-pl.env +``` + +**2. Describe the exit.** In `vpn-pl.env`, fill in `VPN_SERVICE_PROVIDER` and +`VPN_TYPE`, and add the credential lines +[your provider's page](https://github.com/qdm12/gluetun-wiki/tree/main/setup/providers) +lists. + +**3. Use both Compose files** from now on. Add to `.env`: + +```sh +COMPOSE_FILE=compose.yaml:compose.vpn.yaml +``` + +**4. Start, and check the exit's country:** + +```sh +docker compose up -d +docker compose exec vpn-pl wget -qO- https://ipinfo.io/country +``` + +This must print `PL`. TVP VOD already goes through the exit: +`compose.vpn.yaml` sets `MAGNETOWID_TVP_PROXY` to `http://vpn-pl:8888`. + +### A second site + +Copy `vpn-pl.env` to `vpn-gb.env`, set `SERVER_COUNTRIES=United Kingdom` in +the copy, and give it separate credentials if your provider wants them for a +second connection. Then add the exit and the site's setting to +`compose.vpn.yaml`: + +```yaml +services: + magnetowid: + environment: + MAGNETOWID_TVP_PROXY: http://vpn-pl:8888 + MAGNETOWID_BBC_PROXY: http://vpn-gb:8888 + depends_on: + - vpn-pl + - vpn-gb + + vpn-pl: + image: qmcgaw/gluetun:v3 + restart: unless-stopped + cap_add: + - NET_ADMIN + devices: + - /dev/net/tun:/dev/net/tun + env_file: vpn-pl.env + + vpn-gb: + image: qmcgaw/gluetun:v3 + restart: unless-stopped + cap_add: + - NET_ADMIN + devices: + - /dev/net/tun:/dev/net/tun + env_file: vpn-gb.env +``` + +```sh +docker compose up -d +docker compose exec vpn-gb wget -qO- https://ipinfo.io/country +``` + +This must print `GB`. + +## The site settings + +Each takes one of: + +| Value | The site is reached | +|---|---| +| `http://host:port` | through that HTTP proxy | +| `direct` | without a proxy, even if `HTTPS_PROXY` is set | +| not set | as before: through `HTTPS_PROXY` if it is set, otherwise directly | + +The command-line flags are `-tvp-proxy` and `-bbc-proxy`. A site's searches, +streams and subtitles all use its setting. magnetowid logs every site's setting +when it starts. + +While an exit is down, its site's downloads wait as `provider unreachable` +and resume by themselves. They are never sent without the exit. + +## Troubleshooting + +| Problem | What to check | +|---|---| +| The exit doesn't start | `systemctl status magnetowid-vpn@pl` and `journalctl -u magnetowid-vpn@pl -n 50`, or `docker compose logs vpn-pl`. A status of `78/CONFIG` means `VPN_SERVICE_PROVIDER` or `MAGNETOWID_VPN_PORT` is missing from the exit's file. Other errors come from Gluetun and name the setting it rejects: a missing credential, or a country your provider's Gluetun page spells differently. | +| Gluetun rejects the country although it is right | `VPN_SERVICE_PROVIDER` is blank, so Gluetun assumed a provider of its own. | +| The country check fails to connect | The exit is still starting, or the port isn't the exit's `MAGNETOWID_VPN_PORT`. | +| One site's results are empty | Its streams are refused: the log's search lines show `unavailable` above zero. Check that site's setting and its exit's country, then restart magnetowid. | +| Downloads wait as `provider unreachable` | The site's exit is down. Start it; the downloads resume. | +| A download fails with an ffmpeg `Input/output error` | magnetowid fetches streams itself where it can; live and encrypted ones are left to ffmpeg, which can't connect through Gluetun's proxy. These fail instead of bypassing the exit. | diff --git a/internal/provider/bbc/README.md b/internal/provider/bbc/README.md index d2a03b4..58b0b7a 100644 --- a/internal/provider/bbc/README.md +++ b/internal/provider/bbc/README.md @@ -6,7 +6,8 @@ See [Setup and configuration](../../../docs/usage.md) for connection settings. iPlayer streams only to the UK, where watching or downloading BBC programmes requires a TV licence. Elsewhere, searches find programmes but leave every -release out as unavailable. +release out as unavailable; [Region-locked sites](../../../docs/vpn.md) shows +how to give iPlayer a UK connection of its own. ## Series From 5e0f360a9ddfd1511d1de9ce6ac0eb086a1a2edb Mon Sep 17 00:00:00 2001 From: Piotr Komborski <163394+combor@users.noreply.github.com> Date: Thu, 1 Oct 2026 17:36:11 +0100 Subject: [PATCH 4/6] Stop VPN exits without marking them failed Gluetun exits with 1 when it is stopped while its proxy has open connections, so every systemctl stop left the exit failed. Count that status as a stop and restart on any exit instead; an unfinished settings file still ends the unit with 78 and no retries. The smoke test's fake docker now exits with 1 on stop, as Gluetun does. --- cmd/magnetowid/package_smoke_test.go | 13 +++++++++++-- packaging/linux/magnetowid-vpn@.service | 4 +++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/cmd/magnetowid/package_smoke_test.go b/cmd/magnetowid/package_smoke_test.go index 212512c..fb84738 100644 --- a/cmd/magnetowid/package_smoke_test.go +++ b/cmd/magnetowid/package_smoke_test.go @@ -167,8 +167,9 @@ systemctl start magnetowid`) } // A VPN exit runs Gluetun with its instance's settings. Docker is faked: a - // script that records its arguments, behind a unit that does nothing. - c.sh(t, `printf '#!/bin/sh\necho "$@" >>/run/docker.args\n[ "$1" = run ] && exec sleep infinity\nexit 0\n' >/usr/bin/docker + // script that records its arguments, behind a unit that does nothing. Like + // Gluetun, its container exits with 1 when stopped. + c.sh(t, `printf '#!/bin/sh\necho "$@" >>/run/docker.args\nif [ "$1" = run ]; then trap "exit 1" TERM; sleep infinity & wait; fi\nexit 0\n' >/usr/bin/docker chmod +x /usr/bin/docker printf '[Service]\nType=oneshot\nRemainAfterExit=yes\nExecStart=/bin/true\n' >/run/systemd/system/docker.service install -m 600 /usr/share/magnetowid/vpn.env.example /etc/magnetowid/vpn-pl.env @@ -193,6 +194,14 @@ systemctl restart magnetowid-vpn@pl`) t.Errorf("the VPN exit ran docker %s, without %q", run, want) } } + // Stopping it is not a failure, and it comes back if it stops by itself. + c.sh(t, "systemctl stop magnetowid-vpn@pl") + if state, _ := c.try("systemctl is-active magnetowid-vpn@pl"); state != "inactive" { + t.Errorf("stopped VPN exit is %s, want inactive", state) + } + c.sh(t, "systemctl start magnetowid-vpn@pl && kill -TERM $(systemctl show -p MainPID --value magnetowid-vpn@pl)") + c.await(t, "systemctl show -p NRestarts --value magnetowid-vpn@pl", "1") + c.await(t, "systemctl is-active magnetowid-vpn@pl", "active") pid := c.sh(t, "systemctl show -p MainPID --value magnetowid") c.sh(t, d.upgrade) diff --git a/packaging/linux/magnetowid-vpn@.service b/packaging/linux/magnetowid-vpn@.service index 847171d..e9905f4 100644 --- a/packaging/linux/magnetowid-vpn@.service +++ b/packaging/linux/magnetowid-vpn@.service @@ -31,8 +31,10 @@ ExecStart=/bin/sh -c '\ --publish "127.0.0.1:$$MAGNETOWID_VPN_PORT:8888/tcp" \ "$$MAGNETOWID_VPN_IMAGE"' ExecStop=/usr/bin/docker stop magnetowid-vpn-%i -Restart=on-failure +Restart=always RestartSec=10 +# Gluetun exits with 1 when it is stopped, too. +SuccessExitStatus=1 # An exit whose file isn't filled in fails once instead of restarting forever. RestartPreventExitStatus=78 From 4b6eaa3521dbdb81e25cace31774250cebf116f0 Mon Sep 17 00:00:00 2001 From: Piotr Komborski <163394+combor@users.noreply.github.com> Date: Thu, 1 Oct 2026 18:46:32 +0100 Subject: [PATCH 5/6] Leave ffmpeg the environment's proxy rules for a site without a setting A site with no proxy setting got a transport that follows the environment, and the downloader then settled ffmpeg's proxy from the master playlist's URL alone, dropping no_proxy. A rendition or segment on another host could go direct where the environment asked for the proxy. Such a site now has no transport of its own, as before per-site settings: its streams use the default clients and ffmpeg inherits the environment and applies its rules to each host. --- cmd/magnetowid/main.go | 18 +++++++++++------- cmd/magnetowid/main_test.go | 10 +++++----- internal/downloader/ffmpeg.go | 5 +++++ 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/cmd/magnetowid/main.go b/cmd/magnetowid/main.go index f063fcb..b2d76b8 100644 --- a/cmd/magnetowid/main.go +++ b/cmd/magnetowid/main.go @@ -188,16 +188,19 @@ const ( ) // siteClient returns the client for a site's API and streams. proxy is an HTTP -// proxy's URL, "direct", or empty for the environment's HTTPS_PROXY and NO_PROXY. +// proxy's URL or "direct". Left empty, the client has no transport of its own: +// the site and its streams follow the environment's HTTPS_PROXY and NO_PROXY, +// which choose by host, and ffmpeg applies the environment's rules itself. func siteClient(proxy string) (*http.Client, error) { + client := &http.Client{Timeout: siteTimeout} + if proxy == "" { + return client, nil + } t := http.DefaultTransport.(*http.Transport).Clone() // Keep a connection for each of a download's segment workers. t.MaxIdleConnsPerHost = 4 - switch proxy { - case "": - case directProxy: - t.Proxy = nil - default: + t.Proxy = nil + if proxy != directProxy { // ffmpeg, which fetches some streams itself, tunnels through nothing else. u, err := url.Parse(proxy) if err != nil || u.Scheme != "http" || u.Host == "" { @@ -206,7 +209,8 @@ func siteClient(proxy string) (*http.Client, error) { } t.Proxy = http.ProxyURL(u) } - return &http.Client{Timeout: siteTimeout, Transport: t}, nil + client.Transport = t + return client, nil } // proxyName describes a valid proxy setting for the log, without its password. diff --git a/cmd/magnetowid/main_test.go b/cmd/magnetowid/main_test.go index e9e74a4..3011555 100644 --- a/cmd/magnetowid/main_test.go +++ b/cmd/magnetowid/main_test.go @@ -2,7 +2,6 @@ package main import ( "net/http" - "reflect" "strings" "testing" ) @@ -40,9 +39,10 @@ func TestSiteClient(t *testing.T) { return c.Transport.(*http.Transport) } - // The environment's proxy is read once per process, so compare functions. - if got := transport("").Proxy; reflect.ValueOf(got).Pointer() != reflect.ValueOf(http.ProxyFromEnvironment).Pointer() { - t.Error("an unset proxy doesn't follow the environment") + // Unset, the environment decides host by host, for ffmpeg too: a transport + // here would make the downloader settle ffmpeg's proxy from one URL. + if c, err := siteClient(""); err != nil || c.Transport != nil || c.Timeout == 0 { + t.Errorf("siteClient(\"\") = %+v, %v; want a client without a transport", c, err) } if transport("direct").Proxy != nil { t.Error(`"direct" still uses a proxy`) @@ -59,7 +59,7 @@ func TestSiteClient(t *testing.T) { t.Errorf("proxyName(%q) = %q, want %q", proxy, got, name) } } - if a, b := transport(""), transport(""); a == b { + if a, b := transport("direct"), transport("direct"); a == b { t.Error("sites share a transport") } diff --git a/internal/downloader/ffmpeg.go b/internal/downloader/ffmpeg.go index 1eefc57..973d2a0 100644 --- a/internal/downloader/ffmpeg.go +++ b/internal/downloader/ffmpeg.go @@ -127,6 +127,11 @@ func (f *FFmpeg) Download(ctx context.Context, s provider.Stream, out string, pr // proxyEnv returns the environment in which ffmpeg reaches the stream as its // Transport does, or nil, magnetowid's own, for a stream without one. ffmpeg // reads http_proxy for HTTP and HTTPS alike. +// +// The proxy for the stream's URL then serves every host ffmpeg fetches from, +// renditions and segments included, so a Transport must use one proxy for all +// hosts. Proxies chosen by host, as the environment's are, need a stream +// without a Transport: ffmpeg applies the environment's rules itself. func proxyEnv(s provider.Stream) ([]string, error) { t, ok := s.Transport.(*http.Transport) if !ok { From 13dd07b7c89782aab6c15074daa8d9635be46869 Mon Sep 17 00:00:00 2001 From: Piotr Komborski <163394+combor@users.noreply.github.com> Date: Thu, 1 Oct 2026 18:47:32 +0100 Subject: [PATCH 6/6] Keep the API key and proxy passwords out of the usage text Flags took their defaults from the environment, and usage, printed when a required setting is missing, lists every default: the API key and any password in a proxy URL ended up in the service's log. Those flags now default to empty and read the environment after parsing. A flag still wins over the environment. --- cmd/magnetowid/main.go | 12 +++++++++--- cmd/magnetowid/main_test.go | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/cmd/magnetowid/main.go b/cmd/magnetowid/main.go index b2d76b8..42388dd 100644 --- a/cmd/magnetowid/main.go +++ b/cmd/magnetowid/main.go @@ -2,6 +2,7 @@ package main import ( + "cmp" "context" "errors" "flag" @@ -45,16 +46,21 @@ func main() { func run(log *slog.Logger, level *slog.LevelVar) error { listen := flag.String("listen", envOr("MAGNETOWID_LISTEN", ":8484"), "listen address") - apiKey := flag.String("api-key", os.Getenv("MAGNETOWID_API_KEY"), "API key for both APIs (required)") + // Usage prints flag defaults, so settings that hold secrets, the API key and + // a proxy URL's password, get theirs from the environment after parsing. + apiKey := flag.String("api-key", "", "API key for both APIs (required)") downloadDir := flag.String("download-dir", os.Getenv("MAGNETOWID_DOWNLOAD_DIR"), "where finished downloads go (required)") categories := flag.String("categories", envOr("MAGNETOWID_CATEGORIES", "tv,movies"), "comma-separated download categories") ffmpeg := flag.String("ffmpeg", envOr("MAGNETOWID_FFMPEG", "ffmpeg"), "ffmpeg binary") logLevel := flag.String("log-level", envOr("MAGNETOWID_LOG_LEVEL", "info"), "log level: debug, info, warn or error") const proxyUsage = `'s HTTP proxy, e.g. http://127.0.0.1:8888, or "direct" for none (default: the environment's)` - tvpProxy := flag.String("tvp-proxy", os.Getenv("MAGNETOWID_TVP_PROXY"), "TVP VOD"+proxyUsage) - bbcProxy := flag.String("bbc-proxy", os.Getenv("MAGNETOWID_BBC_PROXY"), "BBC iPlayer"+proxyUsage) + tvpProxy := flag.String("tvp-proxy", "", "TVP VOD"+proxyUsage) + bbcProxy := flag.String("bbc-proxy", "", "BBC iPlayer"+proxyUsage) healthcheck := flag.Bool("healthcheck", false, "ask the magnetowid at the listen address whether it is healthy, and exit") flag.Parse() + *apiKey = cmp.Or(*apiKey, os.Getenv("MAGNETOWID_API_KEY")) + *tvpProxy = cmp.Or(*tvpProxy, os.Getenv("MAGNETOWID_TVP_PROXY")) + *bbcProxy = cmp.Or(*bbcProxy, os.Getenv("MAGNETOWID_BBC_PROXY")) if *healthcheck { return checkHealth(*listen) diff --git a/cmd/magnetowid/main_test.go b/cmd/magnetowid/main_test.go index 3011555..a95162b 100644 --- a/cmd/magnetowid/main_test.go +++ b/cmd/magnetowid/main_test.go @@ -1,7 +1,12 @@ package main import ( + "fmt" + "log/slog" "net/http" + "os" + "os/exec" + "slices" "strings" "testing" ) @@ -25,6 +30,35 @@ func TestHealthURL(t *testing.T) { } } +// Usage, printed when a required setting is missing, lists every flag's +// default and ends up in the service's log. The flags belong to the process, +// so a second copy of the test binary runs with secrets in its environment. +func TestUsageHidesSecrets(t *testing.T) { + if os.Getenv("MAGNETOWID_TEST_USAGE") != "" { + err := run(slog.New(slog.DiscardHandler), new(slog.LevelVar)) + fmt.Fprintln(os.Stderr, "run:", err) + return + } + cmd := exec.Command(os.Args[0], "-test.run=^TestUsageHidesSecrets$") + cmd.Env = append(slices.DeleteFunc(os.Environ(), func(kv string) bool { return strings.HasPrefix(kv, "MAGNETOWID_") }), + "MAGNETOWID_TEST_USAGE=1", + "MAGNETOWID_API_KEY=key-secret", + "MAGNETOWID_TVP_PROXY=http://user:tvp-secret@proxy.example:3128", + "MAGNETOWID_BBC_PROXY=http://user:bbc-secret@proxy.example:3128") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("%v\n%s", err, out) + } + usage := string(out) + // The download folder is missing, so the key was read and usage printed. + if !strings.Contains(usage, "-tvp-proxy") || !strings.Contains(usage, "run: -api-key and -download-dir are required") { + t.Fatalf("no usage in the output:\n%s", usage) + } + if strings.Contains(usage, "secret") { + t.Errorf("usage shows a secret:\n%s", usage) + } +} + func TestSiteClient(t *testing.T) { req, err := http.NewRequest(http.MethodGet, "https://vod.example/playlist.m3u8", nil) if err != nil {