From a779149463090c2db3fc6129c22b20e7e319fc44 Mon Sep 17 00:00:00 2001 From: zhengyilei Date: Thu, 25 Jun 2026 19:11:42 +0800 Subject: [PATCH] fix(CubeProxy): disable buffering for envd streaming endpoints Disable nginx response buffering for the verified envd server-streaming endpoints in CubeProxy. This lets early stream frames be delivered promptly for background commands and watch streams, while keeping global buffering behavior unchanged for other routes. Signed-off-by: zhengyilei --- CubeProxy/Dockerfile | 1 + .../includes/envd_streaming_host_route.inc | 24 +++++++ .../includes/envd_streaming_path_route.inc | 28 ++++++++ CubeProxy/nginx.conf | 64 ++++++++++++++++++- 4 files changed, 115 insertions(+), 2 deletions(-) create mode 100644 CubeProxy/conf/includes/envd_streaming_host_route.inc create mode 100644 CubeProxy/conf/includes/envd_streaming_path_route.inc diff --git a/CubeProxy/Dockerfile b/CubeProxy/Dockerfile index c55b70082..5442a623a 100644 --- a/CubeProxy/Dockerfile +++ b/CubeProxy/Dockerfile @@ -11,6 +11,7 @@ RUN sed -i 's#https\?://dl-cdn.alpinelinux.org#http://mirrors.tencent.com#g' /et WORKDIR /usr/local/openresty/nginx/ COPY lua/ /usr/local/openresty/nginx/lua/ +COPY conf/includes/ /usr/local/openresty/nginx/conf/includes/ COPY nginx.conf /usr/local/openresty/nginx/conf/nginx.conf COPY rotate_nginx_log.sh /usr/local/openresty/nginx/sbin/rotate_nginx_log.sh COPY root /etc/crontabs/root diff --git a/CubeProxy/conf/includes/envd_streaming_host_route.inc b/CubeProxy/conf/includes/envd_streaming_host_route.inc new file mode 100644 index 000000000..1e60cb7fd --- /dev/null +++ b/CubeProxy/conf/includes/envd_streaming_host_route.inc @@ -0,0 +1,24 @@ +proxy_buffering off; +# Streaming envd errors must pass through verbatim; intercepting them here +# would replace protocol-native error frames with nginx error pages and break +# client-side parsing. +proxy_intercept_errors off; + +proxy_http_version 1.1; +proxy_set_header Upgrade $http_upgrade; +proxy_set_header Connection "upgrade"; + +proxy_send_timeout 7206s; +proxy_read_timeout 7206s; +proxy_connect_timeout 3s; +proxy_set_header Host $http_host; +proxy_set_header X-Real-IP $remote_addr; +proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + +rewrite_by_lua_file lua/rewrite_phase.lua; + +proxy_pass http://backend; + +header_filter_by_lua_file lua/header_filter_phase.lua; + +log_by_lua_file lua/log_phase.lua; diff --git a/CubeProxy/conf/includes/envd_streaming_path_route.inc b/CubeProxy/conf/includes/envd_streaming_path_route.inc new file mode 100644 index 000000000..88946a2f4 --- /dev/null +++ b/CubeProxy/conf/includes/envd_streaming_path_route.inc @@ -0,0 +1,28 @@ +proxy_buffering off; +# Streaming envd errors must pass through verbatim; intercepting them here +# would replace protocol-native error frames with nginx error pages and break +# client-side parsing. +proxy_intercept_errors off; + +proxy_http_version 1.1; +proxy_set_header Upgrade $http_upgrade; +proxy_set_header Connection "upgrade"; + +proxy_send_timeout 7206s; +proxy_read_timeout 7206s; +proxy_connect_timeout 3s; +proxy_set_header Host $http_host; +proxy_set_header X-Real-IP $remote_addr; +proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; +proxy_set_header X-Forwarded-Prefix /sandbox/$ins_id/$container_port; + +proxy_redirect ~^/(.*)$ /sandbox/$ins_id/$container_port/$1; +proxy_cookie_path / /sandbox/$ins_id/$container_port/; + +rewrite_by_lua_file lua/path_rewrite_phase.lua; + +proxy_pass http://backend; + +header_filter_by_lua_file lua/header_filter_phase.lua; + +log_by_lua_file lua/log_phase.lua; diff --git a/CubeProxy/nginx.conf b/CubeProxy/nginx.conf index 50de7990b..316a684c4 100644 --- a/CubeProxy/nginx.conf +++ b/CubeProxy/nginx.conf @@ -133,9 +133,23 @@ http { proxy_pass http://$cube_sidecar_addr/internal/resume?$args; } + # Path-based server-streaming envd endpoints. Keep buffering enabled + # elsewhere under /sandbox/ and disable it only for these verified + # response streams. + location ~ ^/sandbox/[^/]+/\d+/(?:process\.Process/(?:Start|Connect)|filesystem\.Filesystem/WatchDir)$ { + include /usr/local/openresty/nginx/conf/global/global.conf; + set $garyscale_test "none"; + set $backend_ip ""; + set $backend_port ""; + set $access_time ""; + set $host_proxy_port 8081; + + include /usr/local/openresty/nginx/conf/includes/envd_streaming_path_route.inc; + } + # Path-based routing: /sandbox/// # Lets clients reach a sandbox via a plain IP+port without wildcard DNS or TLS. - location ^~ /sandbox/ { + location /sandbox/ { include /usr/local/openresty/nginx/conf/global/global.conf; set $garyscale_test "none"; set $backend_ip ""; @@ -169,6 +183,25 @@ http { log_by_lua_file lua/log_phase.lua; } + # Verified server-streaming envd endpoints used by clients today. + # Keep buffering enabled elsewhere and disable it only for these + # response streams; otherwise nginx batches early frames until timeout + # or stream completion and breaks immediate-return semantics. + # Current unbuffered streaming endpoints: + # - process.Process/Start + # - process.Process/Connect + # - filesystem.Filesystem/WatchDir + location ~ ^/(?:process\.Process/(?:Start|Connect)|filesystem\.Filesystem/WatchDir)$ { + include /usr/local/openresty/nginx/conf/global/global.conf; + set $garyscale_test "none"; + set $backend_ip ""; + set $backend_port ""; + set $access_time ""; + set $host_proxy_port 8081; + + include /usr/local/openresty/nginx/conf/includes/envd_streaming_host_route.inc; + } + location / { include /usr/local/openresty/nginx/conf/global/global.conf; set $garyscale_test "none"; @@ -226,9 +259,23 @@ http { proxy_pass http://$cube_sidecar_addr/internal/resume?$args; } + # Path-based server-streaming envd endpoints. Keep buffering enabled + # elsewhere under /sandbox/ and disable it only for these verified + # response streams. + location ~ ^/sandbox/[^/]+/\d+/(?:process\.Process/(?:Start|Connect)|filesystem\.Filesystem/WatchDir)$ { + include /usr/local/openresty/nginx/conf/global/global.conf; + set $garyscale_test "none"; + set $backend_ip ""; + set $backend_port ""; + set $access_time ""; + set $host_proxy_port 8080; + + include /usr/local/openresty/nginx/conf/includes/envd_streaming_path_route.inc; + } + # Path-based routing: /sandbox/// # Lets clients reach a sandbox via a plain IP+port without wildcard DNS or TLS. - location ^~ /sandbox/ { + location /sandbox/ { include /usr/local/openresty/nginx/conf/global/global.conf; set $garyscale_test "none"; set $backend_ip ""; @@ -262,6 +309,19 @@ http { log_by_lua_file lua/log_phase.lua; } + # Verified server-streaming envd endpoints. Must not be buffered; see + # the 8081 server block for the scope rationale and endpoint list. + location ~ ^/(?:process\.Process/(?:Start|Connect)|filesystem\.Filesystem/WatchDir)$ { + include /usr/local/openresty/nginx/conf/global/global.conf; + set $garyscale_test "none"; + set $backend_ip ""; + set $backend_port ""; + set $access_time ""; + set $host_proxy_port 8080; + + include /usr/local/openresty/nginx/conf/includes/envd_streaming_host_route.inc; + } + location / { include /usr/local/openresty/nginx/conf/global/global.conf; set $garyscale_test "none";