diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go
index 10a3a13..d1d7be7 100644
--- a/internal/handler/pypi.go
+++ b/internal/handler/pypi.go
@@ -7,19 +7,27 @@ import (
"errors"
"fmt"
"io"
+ "mime"
"net/http"
"net/url"
"regexp"
+ "strconv"
"strings"
"time"
)
const (
- pypiUpstream = "https://pypi.org"
- minWheelParts = 5 // name + version + python + abi + platform
- minSubmatchParts = 2 // full match + first capture group
- minPyPIPathParts = 3 // hash_prefix + hash + filename
- minEggParts = 3 // name + version + python tag
+ pypiUpstream = "https://pypi.org"
+ pypiSimpleJSON = "application/vnd.pypi.simple.v1+json"
+ pypiSimpleHTML = "application/vnd.pypi.simple.v1+html"
+ pypiSimpleLatestJSON = "application/vnd.pypi.simple.latest+json"
+ pypiSimpleLatestHTML = "application/vnd.pypi.simple.latest+html"
+ pypiLegacyHTML = "text/html"
+ pypiExactSpecificity = 2
+ minWheelParts = 5 // name + version + python + abi + platform
+ minSubmatchParts = 2 // full match + first capture group
+ minPyPIPathParts = 3 // hash_prefix + hash + filename
+ minEggParts = 3 // name + version + python tag
// PyPIMetadataSuffix is the PEP 658 core-metadata sidecar suffix that pip
// appends to a distribution URL when the index advertises core metadata.
@@ -49,7 +57,7 @@ func NewPyPIHandler(proxy *Proxy, proxyURL string) *PyPIHandler {
func (h *PyPIHandler) Routes() http.Handler {
mux := http.NewServeMux()
- // Simple API (used by pip)
+ // Simple API
mux.HandleFunc("GET /simple/", h.handleSimpleIndex)
mux.HandleFunc("GET /simple/{name}/", h.handleSimplePackage)
@@ -80,9 +88,10 @@ func (h *PyPIHandler) handleSimplePackage(w http.ResponseWriter, r *http.Request
h.proxy.Logger.Info("pypi simple request", "package", name)
upstreamURL := fmt.Sprintf("%s/simple/%s/", h.upstreamURL, name)
- cacheKey := name + "/simple"
+ accept := selectPyPISimpleRepresentation(r.Header.Get("Accept"))
+ cacheKey := pypiSimpleCacheKey(name, accept)
- body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "pypi", cacheKey, upstreamURL, "text/html")
+ body, contentType, err := h.proxy.FetchOrCacheMetadata(r.Context(), "pypi", cacheKey, upstreamURL, accept)
if err != nil {
if errors.Is(err, ErrUpstreamNotFound) {
http.Error(w, "not found", http.StatusNotFound)
@@ -99,13 +108,131 @@ func (h *PyPIHandler) handleSimplePackage(w http.ResponseWriter, r *http.Request
filteredVersions = h.fetchFilteredVersions(r, name)
}
- rewritten := h.rewriteSimpleHTML(body, filteredVersions)
+ var rewritten []byte
+ if isJSONMediaType(contentType) {
+ rewritten, err = h.rewriteSimpleJSON(body, filteredVersions)
+ if err != nil {
+ h.proxy.Logger.Warn("failed to rewrite pypi simple json, proxying original", "error", err)
+ rewritten = body
+ }
+ } else {
+ rewritten = h.rewriteSimpleHTML(body, filteredVersions)
+ }
- w.Header().Set("Content-Type", "text/html")
+ w.Header().Set("Content-Type", contentType)
+ ensureVaryAccept(w.Header())
w.WriteHeader(http.StatusOK)
_, _ = w.Write(rewritten)
}
+func selectPyPISimpleRepresentation(accept string) string {
+ if strings.TrimSpace(accept) == "" {
+ return pypiLegacyHTML
+ }
+
+ type score struct {
+ quality float64
+ specificity int
+ matched bool
+ }
+
+ scores := map[string]score{
+ pypiSimpleJSON: {},
+ pypiSimpleHTML: {},
+ pypiLegacyHTML: {},
+ }
+
+ update := func(representation string, quality float64, specificity int) {
+ current := scores[representation]
+ if !current.matched || specificity > current.specificity ||
+ (specificity == current.specificity && quality > current.quality) {
+ scores[representation] = score{quality: quality, specificity: specificity, matched: true}
+ }
+ }
+
+ for part := range strings.SplitSeq(accept, ",") {
+ mediaType, params, err := mime.ParseMediaType(strings.TrimSpace(part))
+ if err != nil {
+ continue
+ }
+
+ quality := 1.0
+ if value, ok := params["q"]; ok {
+ quality, err = strconv.ParseFloat(value, 64)
+ if err != nil || quality < 0 || quality > 1 {
+ continue
+ }
+ }
+
+ switch mediaType {
+ case pypiSimpleJSON, pypiSimpleLatestJSON:
+ update(pypiSimpleJSON, quality, pypiExactSpecificity)
+ case pypiSimpleHTML, pypiSimpleLatestHTML:
+ update(pypiSimpleHTML, quality, pypiExactSpecificity)
+ case pypiLegacyHTML:
+ update(pypiLegacyHTML, quality, pypiExactSpecificity)
+ case "application/*":
+ update(pypiSimpleJSON, quality, 1)
+ update(pypiSimpleHTML, quality, 1)
+ case "text/*":
+ update(pypiLegacyHTML, quality, 1)
+ case "*/*":
+ update(pypiSimpleJSON, quality, 0)
+ update(pypiSimpleHTML, quality, 0)
+ update(pypiLegacyHTML, quality, 0)
+ }
+ }
+
+ bestMediaType := ""
+ bestScore := score{}
+ for _, mediaType := range []string{pypiSimpleJSON, pypiSimpleHTML, pypiLegacyHTML} {
+ candidate := scores[mediaType]
+ if !candidate.matched || candidate.quality == 0 {
+ continue
+ }
+ if bestMediaType == "" || candidate.quality > bestScore.quality ||
+ (candidate.quality == bestScore.quality && candidate.specificity > bestScore.specificity) {
+ bestMediaType = mediaType
+ bestScore = candidate
+ }
+ }
+
+ if bestMediaType == "" || bestScore.specificity == 0 {
+ return pypiLegacyHTML
+ }
+ return bestMediaType
+}
+
+func pypiSimpleCacheKey(name, mediaType string) string {
+ switch mediaType {
+ case pypiSimpleJSON:
+ return name + "/simple/json"
+ case pypiSimpleHTML:
+ return name + "/simple/html"
+ default:
+ return name + "/simple"
+ }
+}
+
+func isJSONMediaType(contentType string) bool {
+ mediaType, _, err := mime.ParseMediaType(contentType)
+ if err != nil {
+ return false
+ }
+ return mediaType == "application/json" || strings.HasSuffix(mediaType, "+json")
+}
+
+func ensureVaryAccept(header http.Header) {
+ for _, value := range header.Values("Vary") {
+ for field := range strings.SplitSeq(value, ",") {
+ if strings.EqualFold(strings.TrimSpace(field), "Accept") {
+ return
+ }
+ }
+ }
+ header.Add("Vary", "Accept")
+}
+
// fetchFilteredVersions fetches JSON metadata and returns a set of version strings
// that should be filtered out due to cooldown.
func (h *PyPIHandler) fetchFilteredVersions(r *http.Request, name string) map[string]bool {
@@ -191,6 +318,40 @@ func (h *PyPIHandler) rewriteSimpleHTML(body []byte, filteredVersions map[string
})
}
+func (h *PyPIHandler) rewriteSimpleJSON(body []byte, filteredVersions map[string]bool) ([]byte, error) {
+ var metadata map[string]any
+ if err := json.Unmarshal(body, &metadata); err != nil {
+ return nil, err
+ }
+
+ files, ok := metadata["files"].([]any)
+ if !ok {
+ return nil, errors.New("pypi simple json response has no files array")
+ }
+
+ rewrittenFiles := make([]any, 0, len(files))
+ for _, file := range files {
+ entry, ok := file.(map[string]any)
+ if !ok {
+ rewrittenFiles = append(rewrittenFiles, file)
+ continue
+ }
+
+ if filename, ok := entry["filename"].(string); ok {
+ _, version := h.parseFilename(filename)
+ if version != "" && filteredVersions[version] {
+ continue
+ }
+ }
+
+ h.rewriteURLEntry(entry)
+ rewrittenFiles = append(rewrittenFiles, entry)
+ }
+
+ metadata["files"] = rewrittenFiles
+ return json.Marshal(metadata)
+}
+
// handleJSON serves the JSON API package metadata.
func (h *PyPIHandler) handleJSON(w http.ResponseWriter, r *http.Request) {
name := r.PathValue("name")
@@ -629,7 +790,7 @@ func (h *PyPIHandler) proxySimple(w http.ResponseWriter, r *http.Request, path s
http.Error(w, "failed to create request", http.StatusInternalServerError)
return
}
- req.Header.Set("Accept", "text/html")
+ req.Header.Set("Accept", selectPyPISimpleRepresentation(r.Header.Get("Accept")))
resp, err := h.proxy.HTTPClient.Do(req)
if err != nil {
@@ -644,6 +805,7 @@ func (h *PyPIHandler) proxySimple(w http.ResponseWriter, r *http.Request, path s
w.Header().Add(k, v)
}
}
+ ensureVaryAccept(w.Header())
w.WriteHeader(resp.StatusCode)
_, _ = io.Copy(w, resp.Body)
diff --git a/internal/handler/pypi_test.go b/internal/handler/pypi_test.go
index a416b44..f7df952 100644
--- a/internal/handler/pypi_test.go
+++ b/internal/handler/pypi_test.go
@@ -15,6 +15,267 @@ import (
"github.com/git-pkgs/registries/fetch"
)
+const uvPyPIAccept = "application/vnd.pypi.simple.v1+json, application/vnd.pypi.simple.v1+html;q=0.2, text/html;q=0.01"
+
+type pypiRoundTripFunc func(*http.Request) (*http.Response, error)
+
+func (f pypiRoundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) {
+ return f(r)
+}
+
+func pypiHTTPResponse(r *http.Request, contentType, body string) *http.Response {
+ return &http.Response{
+ StatusCode: http.StatusOK,
+ Status: "200 OK",
+ Header: http.Header{"Content-Type": []string{contentType}},
+ Body: io.NopCloser(strings.NewReader(body)),
+ ContentLength: int64(len(body)),
+ Request: r,
+ }
+}
+
+func setupPyPIHandler(t testing.TB, transport pypiRoundTripFunc) (*PyPIHandler, *Proxy) {
+ t.Helper()
+
+ proxy, _, _, _ := setupTestProxy(t)
+ proxy.HTTPClient = &http.Client{Transport: transport}
+ h := NewPyPIHandler(proxy, "http://proxy.test")
+ h.upstreamURL = "https://pypi.test"
+ return h, proxy
+}
+
+func TestSelectPyPISimpleRepresentation(t *testing.T) {
+ tests := []struct {
+ name string
+ accept string
+ want string
+ }{
+ {"missing header uses legacy html", "", "text/html"},
+ {"wildcard uses legacy html", "*/*", "text/html"},
+ {"uv prefers json", uvPyPIAccept, pypiSimpleJSON},
+ {"json only", pypiSimpleJSON, pypiSimpleJSON},
+ {"latest json", pypiSimpleLatestJSON, pypiSimpleJSON},
+ {"vendor html", pypiSimpleHTML, pypiSimpleHTML},
+ {"higher html quality", pypiSimpleJSON + ";q=0.2, text/html;q=0.8", "text/html"},
+ {"json excluded", pypiSimpleJSON + ";q=0, text/html", "text/html"},
+ {"application wildcard", "application/*", pypiSimpleJSON},
+ {"unsupported type uses legacy html", "application/xml", "text/html"},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if got := selectPyPISimpleRepresentation(tt.accept); got != tt.want {
+ t.Errorf("selectPyPISimpleRepresentation(%q) = %q, want %q", tt.accept, got, tt.want)
+ }
+ })
+ }
+}
+
+func TestPyPISimplePackageNegotiatesJSON(t *testing.T) {
+ const upstreamBody = `{
+ "meta":{"api-version":"1.4"},
+ "name":"ruff",
+ "files":[{
+ "filename":"ruff-0.16.0-py3-none-any.whl",
+ "url":"https://files.pythonhosted.org/packages/ab/cd/ruff-0.16.0-py3-none-any.whl",
+ "hashes":{"sha256":"abc123"},
+ "upload-time":"2026-08-01T12:00:00Z"
+ }]
+ }`
+
+ var upstreamAccept string
+ h, _ := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) {
+ upstreamAccept = r.Header.Get("Accept")
+ if r.URL.Path != "/simple/ruff/" {
+ t.Fatalf("upstream path = %q, want %q", r.URL.Path, "/simple/ruff/")
+ }
+ return pypiHTTPResponse(r, pypiSimpleJSON, upstreamBody), nil
+ })
+
+ req := httptest.NewRequest(http.MethodGet, "/simple/ruff/", nil)
+ req.Header.Set("Accept", uvPyPIAccept)
+ w := httptest.NewRecorder()
+ h.Routes().ServeHTTP(w, req)
+
+ if w.Code != http.StatusOK {
+ t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String())
+ }
+ if upstreamAccept != pypiSimpleJSON {
+ t.Errorf("upstream Accept = %q, want %q", upstreamAccept, pypiSimpleJSON)
+ }
+ if got := w.Header().Get("Content-Type"); got != pypiSimpleJSON {
+ t.Errorf("Content-Type = %q, want %q", got, pypiSimpleJSON)
+ }
+ if got := w.Header().Get("Vary"); !strings.Contains(got, "Accept") {
+ t.Errorf("Vary = %q, want Accept", got)
+ }
+
+ var result struct {
+ Meta map[string]string `json:"meta"`
+ Files []struct {
+ URL string `json:"url"`
+ UploadTime string `json:"upload-time"`
+ } `json:"files"`
+ }
+ if err := json.Unmarshal(w.Body.Bytes(), &result); err != nil {
+ t.Fatalf("decode response: %v", err)
+ }
+ if result.Meta["api-version"] != "1.4" {
+ t.Errorf("api-version = %q, want 1.4", result.Meta["api-version"])
+ }
+ if len(result.Files) != 1 {
+ t.Fatalf("files = %d, want 1", len(result.Files))
+ }
+ if got, want := result.Files[0].URL, "http://proxy.test/pypi/packages/packages/ab/cd/ruff-0.16.0-py3-none-any.whl"; got != want {
+ t.Errorf("file URL = %q, want %q", got, want)
+ }
+ if got := result.Files[0].UploadTime; got != "2026-08-01T12:00:00Z" {
+ t.Errorf("upload-time = %q, want %q", got, "2026-08-01T12:00:00Z")
+ }
+}
+
+func TestPyPISimpleIndexNegotiatesJSON(t *testing.T) {
+ const upstreamBody = `{"meta":{"api-version":"1.4"},"projects":[{"name":"ruff"}]}`
+
+ var upstreamAccept string
+ h, _ := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) {
+ upstreamAccept = r.Header.Get("Accept")
+ return pypiHTTPResponse(r, pypiSimpleJSON, upstreamBody), nil
+ })
+
+ req := httptest.NewRequest(http.MethodGet, "/simple/", nil)
+ req.Header.Set("Accept", uvPyPIAccept)
+ w := httptest.NewRecorder()
+ h.Routes().ServeHTTP(w, req)
+
+ if w.Code != http.StatusOK {
+ t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String())
+ }
+ if upstreamAccept != pypiSimpleJSON {
+ t.Errorf("upstream Accept = %q, want %q", upstreamAccept, pypiSimpleJSON)
+ }
+ if got := w.Header().Get("Content-Type"); got != pypiSimpleJSON {
+ t.Errorf("Content-Type = %q, want %q", got, pypiSimpleJSON)
+ }
+ if got := w.Header().Get("Vary"); !strings.Contains(got, "Accept") {
+ t.Errorf("Vary = %q, want Accept", got)
+ }
+ if got := w.Body.String(); got != upstreamBody {
+ t.Errorf("body = %q, want %q", got, upstreamBody)
+ }
+}
+
+func TestPyPISimplePackageKeepsHTMLDefault(t *testing.T) {
+ const upstreamBody = `ruff-0.16.0.tar.gz`
+
+ var upstreamAccept string
+ h, _ := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) {
+ upstreamAccept = r.Header.Get("Accept")
+ return pypiHTTPResponse(r, "text/html", upstreamBody), nil
+ })
+
+ req := httptest.NewRequest(http.MethodGet, "/simple/ruff/", nil)
+ w := httptest.NewRecorder()
+ h.Routes().ServeHTTP(w, req)
+
+ if w.Code != http.StatusOK {
+ t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String())
+ }
+ if upstreamAccept != "text/html" {
+ t.Errorf("upstream Accept = %q, want text/html", upstreamAccept)
+ }
+ if got := w.Header().Get("Content-Type"); got != "text/html" {
+ t.Errorf("Content-Type = %q, want text/html", got)
+ }
+ if !strings.Contains(w.Body.String(), `href="http://proxy.test/pypi/packages/packages/ab/cd/ruff-0.16.0.tar.gz"`) {
+ t.Errorf("download URL was not rewritten: %s", w.Body.String())
+ }
+}
+
+func TestPyPISimplePackageCachesRepresentationsSeparately(t *testing.T) {
+ hits := make(map[string]int)
+ h, proxy := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) {
+ accept := r.Header.Get("Accept")
+ hits[accept]++
+ if accept == pypiSimpleJSON {
+ body := `{"meta":{"api-version":"1.4"},"name":"ruff","files":[]}`
+ return pypiHTTPResponse(r, pypiSimpleJSON, body), nil
+ }
+ return pypiHTTPResponse(r, accept, `ruff.tar.gz`), nil
+ })
+ proxy.CacheMetadata = true
+ proxy.MetadataTTL = time.Hour
+
+ for range 2 {
+ for _, accept := range []string{pypiLegacyHTML, pypiSimpleHTML, uvPyPIAccept} {
+ req := httptest.NewRequest(http.MethodGet, "/simple/ruff/", nil)
+ req.Header.Set("Accept", accept)
+ w := httptest.NewRecorder()
+ h.Routes().ServeHTTP(w, req)
+ if w.Code != http.StatusOK {
+ t.Fatalf("Accept %q: status = %d, want 200: %s", accept, w.Code, w.Body.String())
+ }
+ }
+ }
+
+ if got := hits[pypiLegacyHTML]; got != 1 {
+ t.Errorf("HTML upstream requests = %d, want 1", got)
+ }
+ if got := hits[pypiSimpleHTML]; got != 1 {
+ t.Errorf("vendor HTML upstream requests = %d, want 1", got)
+ }
+ if got := hits[pypiSimpleJSON]; got != 1 {
+ t.Errorf("JSON upstream requests = %d, want 1", got)
+ }
+}
+
+func TestPyPISimpleJSONCooldown(t *testing.T) {
+ now := time.Now()
+ old := now.Add(-30 * 24 * time.Hour).Format(time.RFC3339)
+ recent := now.Add(-time.Hour).Format(time.RFC3339)
+
+ h, proxy := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) {
+ switch r.URL.Path {
+ case "/simple/ruff/":
+ body := `{"meta":{"api-version":"1.4"},"name":"ruff","files":[` +
+ `{"filename":"ruff-1.0.0.tar.gz","url":"https://files.pythonhosted.org/packages/ab/ruff-1.0.0.tar.gz","upload-time":"` + old + `"},` +
+ `{"filename":"ruff-2.0.0.tar.gz","url":"https://files.pythonhosted.org/packages/cd/ruff-2.0.0.tar.gz","upload-time":"` + recent + `"}` +
+ `]}`
+ return pypiHTTPResponse(r, pypiSimpleJSON, body), nil
+ case "/pypi/ruff/json":
+ body := `{"releases":{` +
+ `"1.0.0":[{"upload_time_iso_8601":"` + old + `"}],` +
+ `"2.0.0":[{"upload_time_iso_8601":"` + recent + `"}]` +
+ `}}`
+ return pypiHTTPResponse(r, "application/json", body), nil
+ default:
+ t.Fatalf("unexpected upstream path: %s", r.URL.Path)
+ return nil, nil
+ }
+ })
+ proxy.Cooldown = &cooldown.Config{Default: "7d"}
+
+ req := httptest.NewRequest(http.MethodGet, "/simple/ruff/", nil)
+ req.Header.Set("Accept", uvPyPIAccept)
+ w := httptest.NewRecorder()
+ h.Routes().ServeHTTP(w, req)
+
+ if w.Code != http.StatusOK {
+ t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String())
+ }
+ var result struct {
+ Files []struct {
+ Filename string `json:"filename"`
+ } `json:"files"`
+ }
+ if err := json.Unmarshal(w.Body.Bytes(), &result); err != nil {
+ t.Fatalf("decode response: %v", err)
+ }
+ if len(result.Files) != 1 || result.Files[0].Filename != "ruff-1.0.0.tar.gz" {
+ t.Errorf("files = %#v, want only ruff-1.0.0.tar.gz", result.Files)
+ }
+}
+
func TestPyPIParseFilename(t *testing.T) {
h := &PyPIHandler{proxy: &Proxy{Logger: slog.Default()}}