feat: HTTPProvider with AutoTLS

.github/workflows/gateway-conformance.yml

@@ -60,11 +60,17 @@ jobs:
         run: make build
         working-directory: kubo-gateway
 
-      # 3. Init the kubo-gateway
+      # 3. Init the kubo-gateway. Every Addresses.* port is set explicitly
+      # and chosen unique across all three conformance jobs (leading 1xxxx
+      # for this one, 2xxxx and 3xxxx for the libp2p and cleartext jobs)
+      # so each job's footprint is unambiguous.
       - name: Init kubo-gateway
         run: |
           ./ipfs init -e
           ./ipfs config --json Gateway.PublicGateways "$GATEWAY_PUBLIC_GATEWAYS"
+          ./ipfs config --json Addresses.Swarm '["/ip4/127.0.0.1/tcp/14001"]'
+          ./ipfs config Addresses.Gateway "/ip4/127.0.0.1/tcp/18080"
+          ./ipfs config Addresses.API "/ip4/127.0.0.1/tcp/15001"
         working-directory: kubo-gateway/cmd/ipfs
 
       # 4. Populate the Kubo gateway with the gateway-conformance fixtures
@@ -95,8 +101,8 @@ jobs:
       - name: Run gateway-conformance tests
         uses: ipfs/gateway-conformance/.github/actions/test@v0.13
         with:
-          gateway-url: http://127.0.0.1:8080
-          subdomain-url: http://localhost:8080
+          gateway-url: http://127.0.0.1:18080
+          subdomain-url: http://localhost:18080
           args: -skip 'TestGatewayCar/GET_response_for_application/vnd.ipld.car/Header_Content-Length'
           json: output.json
           xml: output.xml
@@ -146,14 +152,19 @@ jobs:
         run: make build
         working-directory: kubo-gateway
 
-      # 3. Init the kubo-gateway
+      # 3. Init the kubo-gateway. The trustless subset this job runs does
+      # not exercise subdomain routing, so Gateway.PublicGateways is
+      # unnecessary here. Ports are in the 2xxxx range so each conformance
+      # job's footprint is unambiguous (1xxxx for the full job, 3xxxx for
+      # the cleartext job).
       - name: Init kubo-gateway
         run: |
           ./ipfs init --profile=test
-          ./ipfs config --json Gateway.PublicGateways "$GATEWAY_PUBLIC_GATEWAYS"
-          ./ipfs config --json Experimental.GatewayOverLibp2p true
-          ./ipfs config Addresses.Gateway "/ip4/127.0.0.1/tcp/8080"
-          ./ipfs config Addresses.API "/ip4/127.0.0.1/tcp/5001"
+          ./ipfs config --json HTTPProvider.Enabled true
+          ./ipfs config --json HTTPProvider.Libp2p true
+          ./ipfs config --json Addresses.Swarm '["/ip4/127.0.0.1/tcp/24001"]'
+          ./ipfs config Addresses.Gateway "/ip4/127.0.0.1/tcp/28080"
+          ./ipfs config Addresses.API "/ip4/127.0.0.1/tcp/25001"
         working-directory: kubo-gateway/cmd/ipfs
 
       # 4. Populate the Kubo gateway with the gateway-conformance fixtures
@@ -176,8 +187,9 @@ jobs:
         run: |
           ./ipfs init --profile=test -e
           ./ipfs config --json Experimental.Libp2pStreamMounting true
-          ./ipfs config Addresses.Gateway "/ip4/127.0.0.1/tcp/8081"
-          ./ipfs config Addresses.API "/ip4/127.0.0.1/tcp/5002"
+          ./ipfs config --json Addresses.Swarm '["/ip4/127.0.0.1/tcp/24002"]'
+          ./ipfs config Addresses.Gateway "/ip4/127.0.0.1/tcp/28081"
+          ./ipfs config Addresses.API "/ip4/127.0.0.1/tcp/25002"
         working-directory: kubo-gateway/cmd/ipfs
 
       # 7. Start the kubo http-p2p-proxy
@@ -192,16 +204,16 @@ jobs:
       # 8. Start forwarding data from the http-p2p-proxy to the node serving the Gateway API over libp2p
       - name: Start http-over-libp2p forwarding proxy
         run: |
-          gatewayNodeId=$(./ipfs --api=/ip4/127.0.0.1/tcp/5001 id -f="<id>")
-          ./ipfs --api=/ip4/127.0.0.1/tcp/5002 swarm connect $(./ipfs --api=/ip4/127.0.0.1/tcp/5001 swarm addrs local --id | head -n 1)
-          ./ipfs --api=/ip4/127.0.0.1/tcp/5002 p2p forward --allow-custom-protocol /http/1.1 /ip4/127.0.0.1/tcp/8092 /p2p/$gatewayNodeId
+          gatewayNodeId=$(./ipfs --api=/ip4/127.0.0.1/tcp/25001 id -f="<id>")
+          ./ipfs --api=/ip4/127.0.0.1/tcp/25002 swarm connect $(./ipfs --api=/ip4/127.0.0.1/tcp/25001 swarm addrs local --id | head -n 1)
+          ./ipfs --api=/ip4/127.0.0.1/tcp/25002 p2p forward --allow-custom-protocol /http/1.1 /ip4/127.0.0.1/tcp/28092 /p2p/$gatewayNodeId
         working-directory: kubo-gateway/cmd/ipfs
 
       # 9. Run the gateway-conformance tests over libp2p
       - name: Run gateway-conformance tests over libp2p
         uses: ipfs/gateway-conformance/.github/actions/test@v0.13
         with:
-          gateway-url: http://127.0.0.1:8092
+          gateway-url: http://127.0.0.1:28092
           args: --specs "trustless-gateway,-trustless-ipns-gateway" -skip 'TestGatewayCar/GET_response_for_application/vnd.ipld.car/Header_Content-Length'
           json: output.json
           xml: output.xml
@@ -224,3 +236,94 @@ jobs:
         with:
           name: gateway-conformance-libp2p.json
           path: output.json
+
+  # Testing the trustless gateway subset exposed by HTTPProvider over plain
+  # HTTP/2 (h2c) on the swarm port. HTTPProvider.Cleartext auto-appends a /ws
+  # listener to each /tcp listener in Addresses.Swarm; HTTPProvider then
+  # shares the same TCP port via the shared-TCP demuxer and serves /http
+  # there. No libp2p proxy is needed: the conformance client connects to the
+  # swarm port directly. Complements the libp2p job above so both
+  # HTTPProvider transports are exercised end-to-end.
+  gateway-conformance-http-provider-cleartext:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      # 1. Download the gateway-conformance fixtures
+      - name: Download gateway-conformance fixtures
+        uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.13
+        with:
+          output: fixtures
+
+      # 2. Build the kubo-gateway
+      - name: Checkout kubo-gateway
+        uses: actions/checkout@v6
+        with:
+          path: kubo-gateway
+      - name: Setup Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: 'kubo-gateway/go.mod'
+          cache: true
+          cache-dependency-path: kubo-gateway/go.sum
+      - name: Build kubo-gateway
+        run: make build
+        working-directory: kubo-gateway
+
+      # 3. Init the kubo-gateway with HTTPProvider.Cleartext on the swarm
+      #    port. Libp2p is off; Cleartext is the only HTTPProvider transport
+      #    exercised by this job. The trustless subset run below does not
+      #    exercise subdomain routing, so Gateway.PublicGateways is
+      #    unnecessary here. Ports are in the 3xxxx range so each
+      #    conformance job's footprint is unambiguous (1xxxx for the full
+      #    job, 2xxxx for the libp2p job).
+      - name: Init kubo-gateway
+        run: |
+          ./ipfs init --profile=test
+          ./ipfs config --json HTTPProvider.Enabled true
+          ./ipfs config --json HTTPProvider.Libp2p false
+          ./ipfs config --json HTTPProvider.Cleartext true
+          ./ipfs config --json Addresses.Swarm '["/ip4/127.0.0.1/tcp/34001"]'
+          ./ipfs config Addresses.Gateway "/ip4/127.0.0.1/tcp/38080"
+          ./ipfs config Addresses.API "/ip4/127.0.0.1/tcp/35001"
+        working-directory: kubo-gateway/cmd/ipfs
+
+      # 4. Populate the Kubo gateway with the gateway-conformance fixtures
+      - name: Import fixtures
+        run: |
+          # Import car files
+          find ./fixtures -name '*.car' -exec kubo-gateway/cmd/ipfs/ipfs dag import --pin-roots=false {} \;
+
+      # 5. Start the kubo-gateway
+      - name: Start kubo-gateway
+        run: |
+          ( ./ipfs daemon & ) | sed '/Daemon is ready/q'
+          while [[ "$(./ipfs id | jq '.Addresses | length')" == '0' ]]; do sleep 1; done
+        working-directory: kubo-gateway/cmd/ipfs
+
+      # 6. Run the gateway-conformance tests over plain HTTP/2 (h2c) on the swarm port
+      - name: Run gateway-conformance tests over h2c
+        uses: ipfs/gateway-conformance/.github/actions/test@v0.13
+        with:
+          gateway-url: http://127.0.0.1:34001
+          args: --specs "trustless-gateway,-trustless-ipns-gateway" -skip 'TestGatewayCar/GET_response_for_application/vnd.ipld.car/Header_Content-Length'
+          json: output.json
+          xml: output.xml
+          html: output.html
+          markdown: output.md
+
+      # 7. Upload the results
+      - name: Upload MD summary
+        if: failure() || success()
+        run: cat output.md >> $GITHUB_STEP_SUMMARY
+      - name: Upload HTML report
+        if: failure() || success()
+        uses: actions/upload-artifact@v7
+        with:
+          name: gateway-conformance-http-provider-cleartext.html
+          path: output.html
+      - name: Upload JSON report
+        if: failure() || success()
+        uses: actions/upload-artifact@v7
+        with:
+          name: gateway-conformance-http-provider-cleartext.json
+          path: output.json

.github/workflows/gotest.yml

@@ -202,6 +202,28 @@ jobs:
                 fusermount3 -uz "$mp" 2>/dev/null || fusermount -uz "$mp" 2>/dev/null || true
               done
 
+  # AutoTLS end-to-end canary (in-process Pebble + p2p-forge).
+  # Isolated sub-module (test/autotls); heavy CoreDNS + Pebble deps stay
+  # out of the main go.mod.
+  autotls-tests:
+    if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch'
+    runs-on: ${{ fromJSON(github.repository == 'ipfs/kubo' && '["self-hosted", "linux", "x64", "2xlarge"]' || '"ubuntu-latest"') }}
+    timeout-minutes: 8
+    env:
+      GOTRACEBACK: all
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-go@v6
+        with:
+          go-version-file: 'go.mod'
+      - name: Build kubo binary
+        run: make build
+      - name: Run AutoTLS end-to-end canary
+        run: make test_autotls
+
   # Example tests (kubo-as-a-library)
   example-tests:
     if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch'

.gitignore

@@ -34,6 +34,7 @@ docs/examples/go-ipfs-as-a-library/example-folder/Qm*
 /test/cli/cli-tests.json
 /test/fuse/fuse-unit-tests.json
 /test/fuse/fuse-cli-tests.json
+/test/autotls/autotls-tests.json
 
 # ignore build output from snapcraft
 /ipfs_*.snap

AGENTS.md

@@ -83,7 +83,7 @@ make -O test_go_lint # run linter (use this instead of golangci-lint directly)
 
 If you modify `go.mod` (add/remove/update dependencies), you must run `make mod_tidy` first, before building or testing. Use `make mod_tidy` instead of `go mod tidy` directly, as the project has multiple `go.mod` files.
 
-If you modify any `.go` files outside of `test/`, you must run `make build` before running integration tests.
+If you modify any `.go` files outside of `test/`, you must run `make build` before running integration tests. Integration tests spawn a real `ipfs daemon` from `cmd/ipfs/ipfs` and `go test ./test/cli/...` does not rebuild that binary, so daemon-side changes (config schema, FX providers, transport options, default values) appear to be silently ignored when the binary is stale. Common symptoms: a new config flag has no effect, an expected listener never binds, an FX-injected handler is nil. If a CLI test behaves differently from a manual `cmd/ipfs/ipfs daemon` run, suspect a stale binary first.
 
 ## Testing
 

CHANGELOG.md

@@ -1,5 +1,6 @@
 # Kubo Changelogs
 
+- [v0.44](docs/changelogs/v0.44.md)
 - [v0.43](docs/changelogs/v0.43.md)
 - [v0.42](docs/changelogs/v0.42.md)
 - [v0.41](docs/changelogs/v0.41.md)

Rules.mk

@@ -139,6 +139,7 @@ help:
 	@echo '  test_unit               - Run unit tests with coverage (excludes test/cli)'
 	@echo '  test_cli                - Run CLI integration tests (requires built binary)'
 	@echo '  test_fuse               - Run FUSE tests (requires /dev/fuse and fusermount)'
+	@echo '  test_autotls            - Run AutoTLS end-to-end canary (in-process Pebble + p2p-forge)'
 	@echo '  test_go_fmt             - Check Go source formatting'
 	@echo '  test_go_build           - Build kubo for all platforms from .github/build-platforms.yml'
 	@echo '  test_go_lint            - Run golangci-lint'

cmd/ipfs/kubo/daemon.go

@@ -398,6 +398,17 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment
 		return err
 	}
 
+	// Reject the removed Experimental.GatewayOverLibp2p flag at startup so
+	// stale configs surface a clear migration error instead of silently
+	// running the wrong feature set.
+	if cfg.Experimental.GatewayOverLibp2p {
+		return errors.New(
+			"Experimental.GatewayOverLibp2p has been removed; " +
+				"set HTTPProvider.Enabled=true and HTTPProvider.Libp2p=true instead, " +
+				"then unset Experimental.GatewayOverLibp2p",
+		)
+	}
+
 	// Validate autoconf setup - check for private network conflict
 	swarmKey, _ := repo.SwarmKey()
 	isPrivateNetwork := swarmKey != nil || pnet.ForcePrivateNetwork
@@ -699,8 +710,8 @@ take effect.
 		return err
 	}
 
-	// add trustless gateway over libp2p
-	p2pGwErrc, err := serveTrustlessGatewayOverLibp2p(cctx)
+	// start the HTTPProvider libp2p-stream transport
+	httpProviderErrc, err := serveHTTPProviderOverLibp2p(cctx)
 	if err != nil {
 		return err
 	}
@@ -821,7 +832,7 @@ take effect.
 	// collect long-running errors and block for shutdown
 	// TODO(cryptix): our fuse currently doesn't follow this pattern for graceful shutdown
 	var errs []error
-	for err := range merge(apiErrc, gwErrc, gcErrc, p2pGwErrc, pluginErrc, unmountErrc) {
+	for err := range merge(apiErrc, gwErrc, gcErrc, httpProviderErrc, pluginErrc, unmountErrc) {
 		if err != nil {
 			errs = append(errs, err)
 		}
@@ -1169,22 +1180,55 @@ func serveHTTPGateway(req *cmds.Request, cctx *oldcmds.Context) (<-chan error, e
 	return errc, nil
 }
 
-const gatewayProtocolID protocol.ID = "/ipfs/gateway" // FIXME: specify https://github.com/ipfs/specs/issues/433
+// gatewayProtocolID is the libp2p protocol ID under which the trustless
+// gateway HTTP semantics are exposed over a libp2p stream, per the
+// libp2p Gateway specification:
+// https://specs.ipfs.tech/http-gateways/libp2p-gateway/
+const gatewayProtocolID protocol.ID = "/ipfs/gateway"
+
+// newGatewayWellKnown builds a fresh WellKnownHandler populated with the
+// libp2p Gateway protocol meta. Used to publish /.well-known/libp2p/protocols
+// on the HTTPProvider HTTPS path so clients can discover that the endpoint
+// speaks the libp2p Gateway protocol. The libp2p-stream path uses its own
+// WellKnownHandler attached to p2phttp.Host; both report the same metadata.
+func newGatewayWellKnown() *p2phttp.WellKnownHandler {
+	wk := &p2phttp.WellKnownHandler{}
+	wk.AddProtocolMeta(gatewayProtocolID, p2phttp.ProtocolMeta{Path: "/"})
+	return wk
+}
 
-func serveTrustlessGatewayOverLibp2p(cctx *oldcmds.Context) (<-chan error, error) {
+// serveHTTPProviderOverLibp2p installs the HTTPProvider feature's transports
+// once IpfsNode is fully constructed. It always installs the trustless
+// gateway handler on the AutoWSS-port placeholder (the shared TCP path used
+// by both AutoTLS and HTTPProvider.Cleartext) when HTTPProvider is enabled,
+// and additionally starts the libp2p-stream transport when
+// HTTPProvider.Libp2p is on, mounting the same handler under the
+// /ipfs/gateway libp2p protocol ID per the libp2p Gateway specification
+// (https://specs.ipfs.tech/http-gateways/libp2p-gateway/).
+//
+// The handler itself is the same across transports: NoFetch (only blocks
+// already in the local blockstore), no DNSLink, no HTML errors, raw blocks
+// via ?format=raw only. Deserialized UnixFS responses are not served, so
+// clients always get content-addressed bytes they can verify against the
+// requested CID.
+func serveHTTPProviderOverLibp2p(cctx *oldcmds.Context) (<-chan error, error) {
 	node, err := cctx.ConstructNode()
 	if err != nil {
-		return nil, fmt.Errorf("serveHTTPGatewayOverLibp2p: ConstructNode() failed: %s", err)
+		return nil, fmt.Errorf("serveHTTPProviderOverLibp2p: ConstructNode() failed: %s", err)
 	}
 	cfg, err := node.Repo.Config()
 	if err != nil {
 		return nil, fmt.Errorf("could not read config: %w", err)
 	}
 
-	if !cfg.Experimental.GatewayOverLibp2p {
+	noop := func() <-chan error {
 		errCh := make(chan error)
 		close(errCh)
-		return errCh, nil
+		return errCh
+	}
+
+	if !cfg.HTTPProvider.Enabled.WithDefault(config.DefaultHTTPProviderEnabled) {
+		return noop(), nil
 	}
 
 	opts := []corehttp.ServeOption{
@@ -1198,14 +1242,44 @@ func serveTrustlessGatewayOverLibp2p(cctx *oldcmds.Context) (<-chan error, error
 		return nil, err
 	}
 
+	// Install the trustless handler on the AutoWSS-port placeholder. This
+	// path serves the gateway on the shared TCP port behind any /ws or
+	// /tls/ws listener: AutoTLS-issued /tls/http, manually configured /ws,
+	// and HTTPProvider.Cleartext-derived /ws. The placeholder is provided
+	// by FX whenever HTTPProvider.Enabled is true (see core/node/groups.go),
+	// independently of HTTPProvider.Libp2p below.
+	//
+	// The HTTPS mux also serves /.well-known/libp2p/protocols so HTTPS
+	// clients can discover that this peer speaks /ipfs/gateway (and any
+	// other libp2p protocols mounted in the future), per the libp2p+HTTP
+	// spec (https://specs.ipfs.tech/http-gateways/libp2p-gateway/).
+	//
+	// Wrap with RequireHTTP2OverTLS so the HTTPS path is h2-only (matches
+	// modern public HTTPS endpoints, gives bitswap-httpnet multiplexing)
+	// while the plain /ws path stays permissive for reverse-proxy interop.
+	if node.HTTPProvider != nil {
+		mux := http.NewServeMux()
+		mux.Handle("/", handler)
+		mux.Handle(p2phttp.WellKnownProtocols, newGatewayWellKnown())
+		node.HTTPProvider.Set(libp2p.RequireHTTP2OverTLS(mux))
+		log.Info("HTTPProvider: trustless gateway exposed on /ws and /tls/ws TCP ports (h2 required over TLS, h1+h2c allowed cleartext, .well-known/libp2p/protocols served)")
+	}
+
+	// libp2p-stream transport. Gated by the Libp2p sub-toggle, independent
+	// of the AutoWSS-port install above.
+	if !cfg.HTTPProvider.Libp2p.WithDefault(config.DefaultHTTPProviderLibp2p) {
+		return noop(), nil
+	}
+
 	if node.PeerHost == nil {
 		return nil, fmt.Errorf("cannot create libp2p gateway: node PeerHost is nil (this should not happen and likely indicates an FX dependency injection issue or race condition)")
 	}
 
+	// libp2p-stream gateway. p2phttp.Host.Serve registers
+	// /.well-known/libp2p/protocols on h.ServeMux for us.
 	h := p2phttp.Host{
 		StreamHost: node.PeerHost,
 	}
-
 	h.WellKnownHandler.AddProtocolMeta(gatewayProtocolID, p2phttp.ProtocolMeta{Path: "/"})
 	h.ServeMux = http.NewServeMux()
 	h.ServeMux.Handle("/", handler)