From a9efeec2a8391d2258b93276c48b903d200164df Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Thu, 27 Aug 2026 16:20:31 +0100 Subject: [PATCH] Support PyPI Simple API JSON responses --- internal/handler/pypi.go | 184 ++++++++++++++++++++++-- internal/handler/pypi_test.go | 261 ++++++++++++++++++++++++++++++++++ 2 files changed, 434 insertions(+), 11 deletions(-) diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go index 10a3a13..d1d7be7 100644 --- a/internal/handler/pypi.go +++ b/internal/handler/pypi.go @@ -7,19 +7,27 @@ import ( "errors" "fmt" "io" + "mime" "net/http" "net/url" "regexp" + "strconv" "strings" "time" ) const ( - pypiUpstream = "https://pypi.org" - minWheelParts = 5 // name + version + python + abi + platform - minSubmatchParts = 2 // full match + first capture group - minPyPIPathParts = 3 // hash_prefix + hash + filename - minEggParts = 3 // name + version + python tag + pypiUpstream = "https://pypi.org" + pypiSimpleJSON = "application/vnd.pypi.simple.v1+json" + pypiSimpleHTML = "application/vnd.pypi.simple.v1+html" + pypiSimpleLatestJSON = "application/vnd.pypi.simple.latest+json" + pypiSimpleLatestHTML = "application/vnd.pypi.simple.latest+html" + pypiLegacyHTML = "text/html" + pypiExactSpecificity = 2 + minWheelParts = 5 // name + version + python + abi + platform + minSubmatchParts = 2 // full match + first capture group + minPyPIPathParts = 3 // hash_prefix + hash + filename + minEggParts = 3 // name + version + python tag // PyPIMetadataSuffix is the PEP 658 core-metadata sidecar suffix that pip // appends to a distribution URL when the index advertises core metadata. @@ -49,7 +57,7 @@ func NewPyPIHandler(proxy *Proxy, proxyURL string) *PyPIHandler { func (h *PyPIHandler) Routes() http.Handler { mux := http.NewServeMux() - // Simple API (used by pip) + // Simple API mux.HandleFunc("GET /simple/", h.handleSimpleIndex) mux.HandleFunc("GET /simple/{name}/", h.handleSimplePackage) @@ -80,9 +88,10 @@ func (h *PyPIHandler) handleSimplePackage(w http.ResponseWriter, r *http.Request h.proxy.Logger.Info("pypi simple request", "package", name) upstreamURL := fmt.Sprintf("%s/simple/%s/", h.upstreamURL, name) - cacheKey := name + "/simple" + accept := selectPyPISimpleRepresentation(r.Header.Get("Accept")) + cacheKey := pypiSimpleCacheKey(name, accept) - body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "pypi", cacheKey, upstreamURL, "text/html") + body, contentType, err := h.proxy.FetchOrCacheMetadata(r.Context(), "pypi", cacheKey, upstreamURL, accept) if err != nil { if errors.Is(err, ErrUpstreamNotFound) { http.Error(w, "not found", http.StatusNotFound) @@ -99,13 +108,131 @@ func (h *PyPIHandler) handleSimplePackage(w http.ResponseWriter, r *http.Request filteredVersions = h.fetchFilteredVersions(r, name) } - rewritten := h.rewriteSimpleHTML(body, filteredVersions) + var rewritten []byte + if isJSONMediaType(contentType) { + rewritten, err = h.rewriteSimpleJSON(body, filteredVersions) + if err != nil { + h.proxy.Logger.Warn("failed to rewrite pypi simple json, proxying original", "error", err) + rewritten = body + } + } else { + rewritten = h.rewriteSimpleHTML(body, filteredVersions) + } - w.Header().Set("Content-Type", "text/html") + w.Header().Set("Content-Type", contentType) + ensureVaryAccept(w.Header()) w.WriteHeader(http.StatusOK) _, _ = w.Write(rewritten) } +func selectPyPISimpleRepresentation(accept string) string { + if strings.TrimSpace(accept) == "" { + return pypiLegacyHTML + } + + type score struct { + quality float64 + specificity int + matched bool + } + + scores := map[string]score{ + pypiSimpleJSON: {}, + pypiSimpleHTML: {}, + pypiLegacyHTML: {}, + } + + update := func(representation string, quality float64, specificity int) { + current := scores[representation] + if !current.matched || specificity > current.specificity || + (specificity == current.specificity && quality > current.quality) { + scores[representation] = score{quality: quality, specificity: specificity, matched: true} + } + } + + for part := range strings.SplitSeq(accept, ",") { + mediaType, params, err := mime.ParseMediaType(strings.TrimSpace(part)) + if err != nil { + continue + } + + quality := 1.0 + if value, ok := params["q"]; ok { + quality, err = strconv.ParseFloat(value, 64) + if err != nil || quality < 0 || quality > 1 { + continue + } + } + + switch mediaType { + case pypiSimpleJSON, pypiSimpleLatestJSON: + update(pypiSimpleJSON, quality, pypiExactSpecificity) + case pypiSimpleHTML, pypiSimpleLatestHTML: + update(pypiSimpleHTML, quality, pypiExactSpecificity) + case pypiLegacyHTML: + update(pypiLegacyHTML, quality, pypiExactSpecificity) + case "application/*": + update(pypiSimpleJSON, quality, 1) + update(pypiSimpleHTML, quality, 1) + case "text/*": + update(pypiLegacyHTML, quality, 1) + case "*/*": + update(pypiSimpleJSON, quality, 0) + update(pypiSimpleHTML, quality, 0) + update(pypiLegacyHTML, quality, 0) + } + } + + bestMediaType := "" + bestScore := score{} + for _, mediaType := range []string{pypiSimpleJSON, pypiSimpleHTML, pypiLegacyHTML} { + candidate := scores[mediaType] + if !candidate.matched || candidate.quality == 0 { + continue + } + if bestMediaType == "" || candidate.quality > bestScore.quality || + (candidate.quality == bestScore.quality && candidate.specificity > bestScore.specificity) { + bestMediaType = mediaType + bestScore = candidate + } + } + + if bestMediaType == "" || bestScore.specificity == 0 { + return pypiLegacyHTML + } + return bestMediaType +} + +func pypiSimpleCacheKey(name, mediaType string) string { + switch mediaType { + case pypiSimpleJSON: + return name + "/simple/json" + case pypiSimpleHTML: + return name + "/simple/html" + default: + return name + "/simple" + } +} + +func isJSONMediaType(contentType string) bool { + mediaType, _, err := mime.ParseMediaType(contentType) + if err != nil { + return false + } + return mediaType == "application/json" || strings.HasSuffix(mediaType, "+json") +} + +func ensureVaryAccept(header http.Header) { + for _, value := range header.Values("Vary") { + for field := range strings.SplitSeq(value, ",") { + if strings.EqualFold(strings.TrimSpace(field), "Accept") { + return + } + } + } + header.Add("Vary", "Accept") +} + // fetchFilteredVersions fetches JSON metadata and returns a set of version strings // that should be filtered out due to cooldown. func (h *PyPIHandler) fetchFilteredVersions(r *http.Request, name string) map[string]bool { @@ -191,6 +318,40 @@ func (h *PyPIHandler) rewriteSimpleHTML(body []byte, filteredVersions map[string }) } +func (h *PyPIHandler) rewriteSimpleJSON(body []byte, filteredVersions map[string]bool) ([]byte, error) { + var metadata map[string]any + if err := json.Unmarshal(body, &metadata); err != nil { + return nil, err + } + + files, ok := metadata["files"].([]any) + if !ok { + return nil, errors.New("pypi simple json response has no files array") + } + + rewrittenFiles := make([]any, 0, len(files)) + for _, file := range files { + entry, ok := file.(map[string]any) + if !ok { + rewrittenFiles = append(rewrittenFiles, file) + continue + } + + if filename, ok := entry["filename"].(string); ok { + _, version := h.parseFilename(filename) + if version != "" && filteredVersions[version] { + continue + } + } + + h.rewriteURLEntry(entry) + rewrittenFiles = append(rewrittenFiles, entry) + } + + metadata["files"] = rewrittenFiles + return json.Marshal(metadata) +} + // handleJSON serves the JSON API package metadata. func (h *PyPIHandler) handleJSON(w http.ResponseWriter, r *http.Request) { name := r.PathValue("name") @@ -629,7 +790,7 @@ func (h *PyPIHandler) proxySimple(w http.ResponseWriter, r *http.Request, path s http.Error(w, "failed to create request", http.StatusInternalServerError) return } - req.Header.Set("Accept", "text/html") + req.Header.Set("Accept", selectPyPISimpleRepresentation(r.Header.Get("Accept"))) resp, err := h.proxy.HTTPClient.Do(req) if err != nil { @@ -644,6 +805,7 @@ func (h *PyPIHandler) proxySimple(w http.ResponseWriter, r *http.Request, path s w.Header().Add(k, v) } } + ensureVaryAccept(w.Header()) w.WriteHeader(resp.StatusCode) _, _ = io.Copy(w, resp.Body) diff --git a/internal/handler/pypi_test.go b/internal/handler/pypi_test.go index a416b44..f7df952 100644 --- a/internal/handler/pypi_test.go +++ b/internal/handler/pypi_test.go @@ -15,6 +15,267 @@ import ( "github.com/git-pkgs/registries/fetch" ) +const uvPyPIAccept = "application/vnd.pypi.simple.v1+json, application/vnd.pypi.simple.v1+html;q=0.2, text/html;q=0.01" + +type pypiRoundTripFunc func(*http.Request) (*http.Response, error) + +func (f pypiRoundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { + return f(r) +} + +func pypiHTTPResponse(r *http.Request, contentType, body string) *http.Response { + return &http.Response{ + StatusCode: http.StatusOK, + Status: "200 OK", + Header: http.Header{"Content-Type": []string{contentType}}, + Body: io.NopCloser(strings.NewReader(body)), + ContentLength: int64(len(body)), + Request: r, + } +} + +func setupPyPIHandler(t testing.TB, transport pypiRoundTripFunc) (*PyPIHandler, *Proxy) { + t.Helper() + + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = &http.Client{Transport: transport} + h := NewPyPIHandler(proxy, "http://proxy.test") + h.upstreamURL = "https://pypi.test" + return h, proxy +} + +func TestSelectPyPISimpleRepresentation(t *testing.T) { + tests := []struct { + name string + accept string + want string + }{ + {"missing header uses legacy html", "", "text/html"}, + {"wildcard uses legacy html", "*/*", "text/html"}, + {"uv prefers json", uvPyPIAccept, pypiSimpleJSON}, + {"json only", pypiSimpleJSON, pypiSimpleJSON}, + {"latest json", pypiSimpleLatestJSON, pypiSimpleJSON}, + {"vendor html", pypiSimpleHTML, pypiSimpleHTML}, + {"higher html quality", pypiSimpleJSON + ";q=0.2, text/html;q=0.8", "text/html"}, + {"json excluded", pypiSimpleJSON + ";q=0, text/html", "text/html"}, + {"application wildcard", "application/*", pypiSimpleJSON}, + {"unsupported type uses legacy html", "application/xml", "text/html"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := selectPyPISimpleRepresentation(tt.accept); got != tt.want { + t.Errorf("selectPyPISimpleRepresentation(%q) = %q, want %q", tt.accept, got, tt.want) + } + }) + } +} + +func TestPyPISimplePackageNegotiatesJSON(t *testing.T) { + const upstreamBody = `{ + "meta":{"api-version":"1.4"}, + "name":"ruff", + "files":[{ + "filename":"ruff-0.16.0-py3-none-any.whl", + "url":"https://files.pythonhosted.org/packages/ab/cd/ruff-0.16.0-py3-none-any.whl", + "hashes":{"sha256":"abc123"}, + "upload-time":"2026-08-01T12:00:00Z" + }] + }` + + var upstreamAccept string + h, _ := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) { + upstreamAccept = r.Header.Get("Accept") + if r.URL.Path != "/simple/ruff/" { + t.Fatalf("upstream path = %q, want %q", r.URL.Path, "/simple/ruff/") + } + return pypiHTTPResponse(r, pypiSimpleJSON, upstreamBody), nil + }) + + req := httptest.NewRequest(http.MethodGet, "/simple/ruff/", nil) + req.Header.Set("Accept", uvPyPIAccept) + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String()) + } + if upstreamAccept != pypiSimpleJSON { + t.Errorf("upstream Accept = %q, want %q", upstreamAccept, pypiSimpleJSON) + } + if got := w.Header().Get("Content-Type"); got != pypiSimpleJSON { + t.Errorf("Content-Type = %q, want %q", got, pypiSimpleJSON) + } + if got := w.Header().Get("Vary"); !strings.Contains(got, "Accept") { + t.Errorf("Vary = %q, want Accept", got) + } + + var result struct { + Meta map[string]string `json:"meta"` + Files []struct { + URL string `json:"url"` + UploadTime string `json:"upload-time"` + } `json:"files"` + } + if err := json.Unmarshal(w.Body.Bytes(), &result); err != nil { + t.Fatalf("decode response: %v", err) + } + if result.Meta["api-version"] != "1.4" { + t.Errorf("api-version = %q, want 1.4", result.Meta["api-version"]) + } + if len(result.Files) != 1 { + t.Fatalf("files = %d, want 1", len(result.Files)) + } + if got, want := result.Files[0].URL, "http://proxy.test/pypi/packages/packages/ab/cd/ruff-0.16.0-py3-none-any.whl"; got != want { + t.Errorf("file URL = %q, want %q", got, want) + } + if got := result.Files[0].UploadTime; got != "2026-08-01T12:00:00Z" { + t.Errorf("upload-time = %q, want %q", got, "2026-08-01T12:00:00Z") + } +} + +func TestPyPISimpleIndexNegotiatesJSON(t *testing.T) { + const upstreamBody = `{"meta":{"api-version":"1.4"},"projects":[{"name":"ruff"}]}` + + var upstreamAccept string + h, _ := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) { + upstreamAccept = r.Header.Get("Accept") + return pypiHTTPResponse(r, pypiSimpleJSON, upstreamBody), nil + }) + + req := httptest.NewRequest(http.MethodGet, "/simple/", nil) + req.Header.Set("Accept", uvPyPIAccept) + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String()) + } + if upstreamAccept != pypiSimpleJSON { + t.Errorf("upstream Accept = %q, want %q", upstreamAccept, pypiSimpleJSON) + } + if got := w.Header().Get("Content-Type"); got != pypiSimpleJSON { + t.Errorf("Content-Type = %q, want %q", got, pypiSimpleJSON) + } + if got := w.Header().Get("Vary"); !strings.Contains(got, "Accept") { + t.Errorf("Vary = %q, want Accept", got) + } + if got := w.Body.String(); got != upstreamBody { + t.Errorf("body = %q, want %q", got, upstreamBody) + } +} + +func TestPyPISimplePackageKeepsHTMLDefault(t *testing.T) { + const upstreamBody = `ruff-0.16.0.tar.gz` + + var upstreamAccept string + h, _ := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) { + upstreamAccept = r.Header.Get("Accept") + return pypiHTTPResponse(r, "text/html", upstreamBody), nil + }) + + req := httptest.NewRequest(http.MethodGet, "/simple/ruff/", nil) + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String()) + } + if upstreamAccept != "text/html" { + t.Errorf("upstream Accept = %q, want text/html", upstreamAccept) + } + if got := w.Header().Get("Content-Type"); got != "text/html" { + t.Errorf("Content-Type = %q, want text/html", got) + } + if !strings.Contains(w.Body.String(), `href="http://proxy.test/pypi/packages/packages/ab/cd/ruff-0.16.0.tar.gz"`) { + t.Errorf("download URL was not rewritten: %s", w.Body.String()) + } +} + +func TestPyPISimplePackageCachesRepresentationsSeparately(t *testing.T) { + hits := make(map[string]int) + h, proxy := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) { + accept := r.Header.Get("Accept") + hits[accept]++ + if accept == pypiSimpleJSON { + body := `{"meta":{"api-version":"1.4"},"name":"ruff","files":[]}` + return pypiHTTPResponse(r, pypiSimpleJSON, body), nil + } + return pypiHTTPResponse(r, accept, `ruff.tar.gz`), nil + }) + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + + for range 2 { + for _, accept := range []string{pypiLegacyHTML, pypiSimpleHTML, uvPyPIAccept} { + req := httptest.NewRequest(http.MethodGet, "/simple/ruff/", nil) + req.Header.Set("Accept", accept) + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("Accept %q: status = %d, want 200: %s", accept, w.Code, w.Body.String()) + } + } + } + + if got := hits[pypiLegacyHTML]; got != 1 { + t.Errorf("HTML upstream requests = %d, want 1", got) + } + if got := hits[pypiSimpleHTML]; got != 1 { + t.Errorf("vendor HTML upstream requests = %d, want 1", got) + } + if got := hits[pypiSimpleJSON]; got != 1 { + t.Errorf("JSON upstream requests = %d, want 1", got) + } +} + +func TestPyPISimpleJSONCooldown(t *testing.T) { + now := time.Now() + old := now.Add(-30 * 24 * time.Hour).Format(time.RFC3339) + recent := now.Add(-time.Hour).Format(time.RFC3339) + + h, proxy := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) { + switch r.URL.Path { + case "/simple/ruff/": + body := `{"meta":{"api-version":"1.4"},"name":"ruff","files":[` + + `{"filename":"ruff-1.0.0.tar.gz","url":"https://files.pythonhosted.org/packages/ab/ruff-1.0.0.tar.gz","upload-time":"` + old + `"},` + + `{"filename":"ruff-2.0.0.tar.gz","url":"https://files.pythonhosted.org/packages/cd/ruff-2.0.0.tar.gz","upload-time":"` + recent + `"}` + + `]}` + return pypiHTTPResponse(r, pypiSimpleJSON, body), nil + case "/pypi/ruff/json": + body := `{"releases":{` + + `"1.0.0":[{"upload_time_iso_8601":"` + old + `"}],` + + `"2.0.0":[{"upload_time_iso_8601":"` + recent + `"}]` + + `}}` + return pypiHTTPResponse(r, "application/json", body), nil + default: + t.Fatalf("unexpected upstream path: %s", r.URL.Path) + return nil, nil + } + }) + proxy.Cooldown = &cooldown.Config{Default: "7d"} + + req := httptest.NewRequest(http.MethodGet, "/simple/ruff/", nil) + req.Header.Set("Accept", uvPyPIAccept) + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String()) + } + var result struct { + Files []struct { + Filename string `json:"filename"` + } `json:"files"` + } + if err := json.Unmarshal(w.Body.Bytes(), &result); err != nil { + t.Fatalf("decode response: %v", err) + } + if len(result.Files) != 1 || result.Files[0].Filename != "ruff-1.0.0.tar.gz" { + t.Errorf("files = %#v, want only ruff-1.0.0.tar.gz", result.Files) + } +} + func TestPyPIParseFilename(t *testing.T) { h := &PyPIHandler{proxy: &Proxy{Logger: slog.Default()}}