Add https proxy support

vertx-core/src/main/asciidoc/http.adoc

@@ -2407,7 +2407,8 @@ used with binary frames that are no split over multiple frames.
 
 === Using a proxy for HTTP/HTTPS connections
 
-The {@link io.vertx.core.http.HttpClient} supports accessing HTTP/HTTPS URLs via an HTTP proxy (e.g. Squid), a _SOCKS4a_, or a _SOCKS5_ proxy.
+The {@link io.vertx.core.http.HttpClient} supports accessing HTTP/HTTPS URLs via an HTTP proxy (e.g. Squid), an HTTPS proxy
+(an HTTP proxy reached over SSL/TLS), a _SOCKS4a_, or a _SOCKS5_ proxy.
 The `CONNECT` protocol uses HTTP/1.x but can connect to HTTP/1.x and HTTP/2 servers.
 
 Connecting to `h2c` (unencrypted HTTP/2 servers) is likely not supported by http proxies since they will support HTTP/1.1 only.
@@ -2426,6 +2427,11 @@ When the client connects to an HTTP URL, it connects to the proxy server and pro
 
 When the client connects to an HTTPS URL, it asks the proxy to create a tunnel to the remote host with the `CONNECT` method.
 
+To connect to the proxy itself over SSL/TLS, set the proxy type to {@link io.vertx.core.net.ProxyType#HTTPS} and configure
+the proxy SSL options with {@link io.vertx.core.net.ProxyOptions#setSslOptions(io.vertx.core.net.ClientSSLOptions)}. The
+proxy SSL options are independent of the target server's SSL options, and hostname verification of the proxy certificate is
+enabled by default.
+
 For a SOCKS5 proxy:
 
 [source,$lang]

vertx-core/src/main/asciidoc/tcp.adoc

@@ -506,11 +506,19 @@ the update.
 
 === Using a proxy for client connections
 
-The {@link io.vertx.core.net.NetClient} supports either an HTTP/1.x _CONNECT_, _SOCKS4a_ or _SOCKS5_ proxy.
+The {@link io.vertx.core.net.NetClient} supports either an HTTP/1.x _CONNECT_, an HTTPS (HTTP _CONNECT_ over SSL/TLS),
+_SOCKS4a_ or _SOCKS5_ proxy.
 
 The proxy can be configured in the {@link io.vertx.core.net.TcpClientConfig} by setting a
 {@link io.vertx.core.net.ProxyOptions} object containing proxy type, hostname, port and optionally username and password.
 
+To reach the proxy itself over SSL/TLS, use the {@link io.vertx.core.net.ProxyType#HTTPS} proxy type and configure the
+SSL options for the proxy connection with {@link io.vertx.core.net.ProxyOptions#setSslOptions(io.vertx.core.net.ClientSSLOptions)}.
+These options (trust store, optional client certificate, hostname verification) apply to the connection to the proxy and
+are independent of the options used for the target server. Hostname verification of the proxy certificate is enabled by
+default. Note that {@link io.vertx.core.net.ProxyType#HTTPS} denotes a proxy that is itself reached over SSL/TLS, which is
+distinct from the `https_proxy` environment-variable convention of a proxy used for `https` traffic.
+
 Here's an example:
 
 [source,$lang]

vertx-core/src/main/generated/io/vertx/core/net/ProxyOptionsConverter.java

@@ -47,6 +47,11 @@ static void fromJson(Iterable<java.util.Map.Entry<String, Object>> json, ProxyOp
             obj.setConnectTimeout(java.time.Duration.of(((Number)member.getValue()).longValue(), java.time.temporal.ChronoUnit.MILLIS));
           }
           break;
+        case "sslOptions":
+          if (member.getValue() instanceof JsonObject) {
+            obj.setSslOptions(new io.vertx.core.net.ClientSSLOptions((io.vertx.core.json.JsonObject)member.getValue()));
+          }
+          break;
       }
     }
   }
@@ -75,5 +80,8 @@ static void toJson(ProxyOptions obj, java.util.Map<String, Object> json) {
     if (obj.getConnectTimeout() != null) {
       json.put("connectTimeout", obj.getConnectTimeout().toMillis());
     }
+    if (obj.getSslOptions() != null) {
+      json.put("sslOptions", obj.getSslOptions().toJson());
+    }
   }
 }

vertx-core/src/main/java/io/vertx/core/http/impl/EndpointKey.java

@@ -79,14 +79,15 @@ private static boolean equals(ProxyOptions options1, ProxyOptions options2) {
         Objects.equals(options1.getType(), options2.getType()) &&
         Objects.equals(options1.getUsername(), options2.getUsername()) &&
         Objects.equals(options1.getPassword(), options2.getPassword()) &&
-        Objects.equals(options1.getProxyAuthorization(), options2.getProxyAuthorization());
+        Objects.equals(options1.getProxyAuthorization(), options2.getProxyAuthorization()) &&
+        Objects.equals(options1.getSslOptions(), options2.getSslOptions());
     }
     return false;
   }
 
   private static int hashCode(ProxyOptions options) {
     return Objects.hash(options.getHost(), options.getPort(), options.getType(), options.getUsername(),
-      options.getPassword(), options.getProxyAuthorization());
+      options.getPassword(), options.getProxyAuthorization(), options.getSslOptions());
   }
 
   @Override

vertx-core/src/main/java/io/vertx/core/http/impl/HttpClientImpl.java

@@ -191,10 +191,13 @@ private Function<EndpointKey, SharedHttpClientConnectionGroup> httpEndpointProvi
       PoolMetrics poolMetrics = HttpClientImpl.this.httpMetrics != null ? vertx.metrics().createPoolMetrics("http", key.authority.toString(), maxPoolSize) : null;
       ProxyOptions proxyOptions = key.proxyOptions;
       ClientSSLOptions sslOptions = key.sslOptions;
-      if (proxyOptions != null && !key.ssl && proxyOptions.getType() == ProxyType.HTTP) {
+      boolean forwardProxy = false;
+      if (proxyOptions != null && !key.ssl && (proxyOptions.getType() == ProxyType.HTTP || proxyOptions.getType() == ProxyType.HTTPS)) {
         SocketAddress server = SocketAddress.inetSocketAddress(proxyOptions.getPort(), proxyOptions.getHost());
         key = new EndpointKey(key.ssl, key.protocol, sslOptions, proxyOptions, server, key.authority);
-        proxyOptions = null;
+        // Forward mode: the single socket connects to the proxy as the server while the logical
+        // origin stays plain HTTP (key.ssl == false).
+        forwardProxy = true;
       }
       HttpVersion protocol = key.protocol;
       List<HttpVersion> protocols;
@@ -203,7 +206,7 @@ private Function<EndpointKey, SharedHttpClientConnectionGroup> httpEndpointProvi
       } else {
         protocols = List.of(protocol);
       }
-      HttpConnectParams params = new HttpConnectParams(protocols, sslOptions, proxyOptions, key.ssl);
+      HttpConnectParams params = new HttpConnectParams(protocols, sslOptions, proxyOptions, key.ssl, forwardProxy);
       Function<SharedHttpClientConnectionGroup, SharedHttpClientConnectionGroup.Pool> p = group -> {
         int queueMaxSize = poolOptions.getMaxWaitQueueSize();
         if (transport instanceof TcpHttpClientTransport) {

vertx-core/src/main/java/io/vertx/core/http/impl/HttpClientRequestImpl.java

@@ -89,7 +89,7 @@ public void init(RequestOptions options) {
     Boolean followRedirects = options.getFollowRedirects();
     long idleTimeout = options.getIdleTimeout();
     ProxyOptions proxyOptions = options.getProxyOptions();
-    if (proxyOptions != null && !useSSL && proxyOptions.getType() == ProxyType.HTTP) {
+    if (proxyOptions != null && !useSSL && (proxyOptions.getType() == ProxyType.HTTP || proxyOptions.getType() == ProxyType.HTTPS)) {
       HostAndPort authority = conn.authority();
       if (!ABS_URI_START_PATTERN.matcher(requestURI).find()) {
         int defaultPort = 80;

vertx-core/src/main/java/io/vertx/core/http/impl/HttpConnectParams.java

@@ -12,15 +12,27 @@ public HttpConnectParams(List<HttpVersion> protocols,
                            ClientSSLOptions sslOptions,
                            ProxyOptions proxyOptions,
                            boolean ssl) {
+    this(protocols, sslOptions, proxyOptions, ssl, false);
+  }
+
+  public HttpConnectParams(List<HttpVersion> protocols,
+                           ClientSSLOptions sslOptions,
+                           ProxyOptions proxyOptions,
+                           boolean ssl,
+                           boolean forwardProxy) {
     this.protocols = protocols;
     this.sslOptions = sslOptions;
     this.proxyOptions = proxyOptions;
     this.ssl = ssl;
+    this.forwardProxy = forwardProxy;
   }
 
   public final List<HttpVersion> protocols;
   public final ClientSSLOptions sslOptions;
   public final ProxyOptions proxyOptions;
+
   public final boolean ssl;
 
+  public final boolean forwardProxy;
+
 }

vertx-core/src/main/java/io/vertx/core/http/impl/http1/Http1ClientConnection.java

@@ -149,6 +149,13 @@ public HostAndPort authority() {
     return authority;
   }
 
+  @Override
+  public boolean isSsl() {
+    // Report the origin ssl, not the pipeline: in forward-proxy mode the TLS leg to an HTTPS proxy is
+    // itself the pipeline "ssl" handler.
+    return ssl;
+  }
+
   @Override
   public io.vertx.core.http.impl.HttpClientConnection evictionHandler(Handler<Void> handler) {
     evictionHandler = handler;

vertx-core/src/main/java/io/vertx/core/http/impl/tcp/TcpHttpClientTransport.java

@@ -143,26 +143,57 @@ private Http2ClientChannelInitializer http2Initializer() {
     }
   }
 
+  /**
+   * Whether this connection's transport TLS terminates at an HTTPS proxy (forward mode through an
+   * HTTPS proxy) — the only forward case that uses leg-1 TLS to the proxy.
+   */
+  private static boolean proxyHttps(HttpConnectParams params) {
+    return params.forwardProxy && params.proxyOptions != null && params.proxyOptions.getType() == ProxyType.HTTPS;
+  }
+
+  /** Whether the socket is encrypted: origin TLS, or leg-1 TLS to an HTTPS proxy (forward mode). */
+  private static boolean transportSsl(HttpConnectParams params) {
+    return params.ssl || proxyHttps(params);
+  }
+
   private void connect(ContextInternal context, HttpConnectParams params, HostAndPort authority, SocketAddress server, Promise<NetSocket> promise) {
+    boolean forward = params.forwardProxy;
+    boolean proxyHttps = proxyHttps(params);
+    boolean transportSsl = transportSsl(params);
+
     ConnectOptions connectOptions = new ConnectOptions();
     connectOptions.setRemoteAddress(server);
-    if (authority != null) {
+    if (proxyHttps) {
+      // Forward mode through an HTTPS proxy: the socket's TLS terminates at the proxy, so the peer /
+      // SNI is the proxy (the server), not the origin authority.
+      connectOptions.setHost(server.host());
+      connectOptions.setPort(server.port());
+      connectOptions.setSniServerName(server.host());
+    } else if (authority != null) {
       connectOptions.setHost(authority.host());
       connectOptions.setPort(authority.port());
-      if (params.ssl && forceSni) {
+      if (transportSsl && forceSni) {
         connectOptions.setSniServerName(authority.host());
       }
     }
-    connectOptions.setSsl(params.ssl);
-    if (params.ssl) {
+    connectOptions.setSsl(transportSsl);
+    if (transportSsl) {
+      // The TLS being configured terminates at the proxy (forward + HTTPS proxy) or at the origin
+      // (direct, or the leg-2 of a CONNECT tunnel); pick the matching options.
+      ClientSSLOptions transportSslOptions = proxyHttps ? params.proxyOptions.getSslOptions() : params.sslOptions;
       ClientSSLOptions copy;
-      if (params.sslOptions != null) {
-        copy = params.sslOptions.copy();
+      if (transportSslOptions != null) {
+        copy = transportSslOptions.copy();
       } else {
         // We might end up using javax.net.ssl.trustStore
         copy = new ClientSSLOptions().setHostnameVerificationAlgorithm("HTTPS");
       }
-      boolean useAlpn = params.protocols.contains(HttpVersion.HTTP_2);
+      if (proxyHttps && copy.getHostnameVerificationAlgorithm() == null) {
+        // Verify the proxy certificate against the proxy host by default (secure default).
+        copy.setHostnameVerificationAlgorithm("HTTPS");
+      }
+      // No ALPN on a leg-1 TLS connection to the proxy: ALPN belongs to the origin (leg 2).
+      boolean useAlpn = !proxyHttps && params.protocols.contains(HttpVersion.HTTP_2);
       copy.setUseAlpn(useAlpn);
       if (useAlpn) {
         List<String> list = params.protocols
@@ -174,7 +205,9 @@ private void connect(ContextInternal context, HttpConnectParams params, HostAndP
       }
       connectOptions.setSslOptions(copy);
     }
-    connectOptions.setProxyOptions(params.proxyOptions);
+    // Only CONNECT/SOCKS-tunnel through the proxy when NOT forwarding; in forward mode the socket
+    // connects straight to the proxy (the server) and issues absolute-URI requests.
+    connectOptions.setProxyOptions(forward ? null : params.proxyOptions);
     client.connectInternal(connectOptions, promise, context);
   }
 
@@ -197,7 +230,11 @@ public Future<HttpClientConnection> wrap(ContextInternal context, HttpConnectPar
 
     //
     Channel ch = so.channelHandlerContext().channel();
-    if (params.ssl) {
+    // Transport TLS may be on because the origin is HTTPS or because leg-1 TLS to an HTTPS proxy is
+    // in use (forward mode). The connection's ssl flag, however, must reflect the logical origin
+    // (params.ssl), not the transport.
+    boolean transportSsl = transportSsl(params);
+    if (transportSsl) {
       String protocol = so.applicationLayerProtocol();
       if (protocol == null) {
         protocol = "";
@@ -218,7 +255,7 @@ public Future<HttpClientConnection> wrap(ContextInternal context, HttpConnectPar
           if (http1Config != null) {
             applyHttp1xConnectionOptions(ch.pipeline());
             HttpVersion version = "http/1.0".equals(protocol) ? HttpVersion.HTTP_1_0 : HttpVersion.HTTP_1_1;
-            http1xConnected(version, server, authority, true, context, transportMetrics, metric, ch, clientMetrics, promise);
+            http1xConnected(version, server, authority, params.ssl, context, transportMetrics, metric, ch, clientMetrics, promise);
           } else {
             so.close();
             promise.tryFail(new IllegalStateException("HTTP/1.1 not supported"));

vertx-core/src/main/java/io/vertx/core/net/ProxyOptions.java

@@ -56,6 +56,7 @@ public class ProxyOptions {
   private String proxyAuthorization;
   private ProxyType type;
   private Duration connectTimeout;
+  private ClientSSLOptions sslOptions;
 
   /**
    * Default constructor.
@@ -80,6 +81,7 @@ public ProxyOptions(ProxyOptions other) {
     proxyAuthorization = other.getProxyAuthorization();
     type = other.getType();
     connectTimeout = other.getConnectTimeout();
+    sslOptions = other.sslOptions != null ? other.sslOptions.copy() : null;
   }
 
   /**
@@ -260,4 +262,32 @@ public ProxyOptions setConnectTimeout(Duration connectTimeout) {
     this.connectTimeout = connectTimeout;
     return this;
   }
+
+  /**
+   * Get the SSL options used between the client and the proxy.
+   * <p>
+   * Only relevant when {@link #getType()} is {@link ProxyType#HTTPS}.
+   *
+   * @return the proxy SSL options, or {@code null}
+   */
+  public ClientSSLOptions getSslOptions() {
+    return sslOptions;
+  }
+
+  /**
+   * Set the SSL options used for the connection to the proxy itself, when the proxy is reached over
+   * SSL/TLS ({@link ProxyType#HTTPS}).
+   * <p>
+   * These options are resolved independently of the origin SSL options, so the proxy presents and
+   * is validated against its own certificate and trust store. When left unset for an
+   * {@link ProxyType#HTTPS} proxy, the connection is still established over SSL/TLS using the default
+   * trust source, with hostname verification enabled against the proxy host.
+   *
+   * @param sslOptions the proxy SSL options
+   * @return a reference to this, so the API can be used fluently
+   */
+  public ProxyOptions setSslOptions(ClientSSLOptions sslOptions) {
+    this.sslOptions = sslOptions;
+    return this;
+  }
 }

vertx-core/src/main/java/io/vertx/core/net/ProxyType.java

@@ -24,6 +24,16 @@ public enum ProxyType {
    * HTTP CONNECT ssl proxy
    */
   HTTP,
+  /**
+   * HTTP proxy reached over SSL/TLS, i.e. the client opens an {@code https} connection to the proxy
+   * itself before issuing {@code CONNECT} or absolute-URI requests. The proxying semantics are
+   * identical to {@link #HTTP}; only the connection to the proxy is encrypted, configured via
+   * {@link ProxyOptions#setSslOptions(ClientSSLOptions)}.
+   * <p>
+   * Note this is distinct from the {@code https_proxy} environment-variable convention, which
+   * denotes the proxy used <em>for</em> {@code https} traffic rather than a proxy reached over SSL/TLS.
+   */
+  HTTPS,
   /**
    * SOCKS4/4a tcp proxy
    */

vertx-core/src/main/java/io/vertx/core/net/impl/ConnectionBase.java

@@ -301,7 +301,18 @@ public void flushBytesWritten() {
   }
 
   public boolean isSsl() {
-    return chctx.pipeline().get(SslHandler.class) != null;
+    // Ignore a TLS leg to an HTTPS proxy ("proxy-ssl"): report whether the origin connection is encrypted.
+    ChannelHandler proxySsl = chctx.pipeline().get("proxy-ssl");
+    if (proxySsl == null) {
+      return chctx.pipeline().get(SslHandler.class) != null;
+    }
+    for (String name : chctx.pipeline().names()) {
+      ChannelHandler handler = chctx.pipeline().get(name);
+      if (handler != proxySsl && handler instanceof SslHandler) {
+        return true;
+      }
+    }
+    return false;
   }
 
   public boolean isTrafficShaped() {

vertx-core/src/main/java/io/vertx/core/net/impl/VertxConnection.java

@@ -12,6 +12,7 @@
 
 import io.netty.buffer.Unpooled;
 import io.netty.channel.*;
+import io.netty.handler.ssl.SslHandler;
 import io.netty.handler.timeout.IdleStateEvent;
 import io.netty.util.ReferenceCountUtil;
 import io.netty.util.ReferenceCounted;
@@ -141,7 +142,8 @@ protected void handleIdle(IdleStateEvent event) {
   }
 
   protected boolean supportsFileRegion() {
-    return vertx.transport().supportFileRegion() && !isSsl() &&!isTrafficShaped();
+    boolean sslChannel = chctx.pipeline().get(SslHandler.class) != null;
+    return vertx.transport().supportFileRegion() && !sslChannel && !isTrafficShaped();
   }
 
   /**