From c7dd3aad135ae272efcd844f9a3d65054a859529 Mon Sep 17 00:00:00 2001 From: Tomas Srnka Date: Mon, 14 Sep 2026 18:22:34 +0000 Subject: [PATCH 1/2] fix(js-sdk): send the upload-request headers the API returns MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The file-upload-link response carries `headers` — request headers the upload must send that the signed URL cannot carry itself. The API sets them when the storage backend needs them: Azure signs layer-file uploads with a SAS and returns `x-ms-blob-type: BlockBlob`, which a SAS cannot express. `TemplateBase.build` destructured only `{ present, url }` and `putFileStream` PUT with nothing but `Content-Length`, so on an Azure-backed cluster every uncached `COPY` failed with a storage `400 MissingRequiredHeader`. GCS and S3 return no headers, which is why the live-API suites never saw it. Thread the headers into the PUT. The map is an open `additionalProperties: string`, so spreading it under `Content-Length` does not protect body framing: a lowercase `content-length` from the API survives the exact-case shadowing as a second key, and undici joins the pair into `5, 11` and rejects the request. Drop `content-length` and `transfer-encoding` case-insensitively before merging, then apply the archive's own `Content-Length` last. A missing or empty map behaves exactly as before. The upload error wrap now carries `error.cause`: fetch reports transport failures as a bare `TypeError: fetch failed`, which told the user nothing. Verification: `tests/template/uploadHeaders.test.ts` drives `Template.buildInBackground` against a mocked upload link and a local PUT server, asserting the header arrives with headers set and is absent without; `uploadFile.test.ts` covers the `buildApi` boundary and adds a row where the API returns `content-length: 1`, asserting the server saw the real archive size, one `content-length` header and no chunked encoding. All fail without this change. The msw server now errors on unhandled requests rather than letting them escape to the real API. --- .changeset/template-upload-request-headers.md | 6 + packages/js-sdk/src/template/buildApi.ts | 31 ++++- packages/js-sdk/src/template/index.ts | 3 +- .../js-sdk/tests/template/uploadFile.test.ts | 53 ++++++++ .../tests/template/uploadHeaders.test.ts | 118 ++++++++++++++++++ 5 files changed, 207 insertions(+), 4 deletions(-) create mode 100644 .changeset/template-upload-request-headers.md create mode 100644 packages/js-sdk/tests/template/uploadHeaders.test.ts diff --git a/.changeset/template-upload-request-headers.md b/.changeset/template-upload-request-headers.md new file mode 100644 index 0000000000..f9f3b8e0f8 --- /dev/null +++ b/.changeset/template-upload-request-headers.md @@ -0,0 +1,6 @@ +--- +'e2b': patch +'@e2b/python-sdk': patch +--- + +Template file uploads now send the request headers the file-upload-link response returns. Azure-backed clusters sign layer-file uploads with a SAS and return `x-ms-blob-type: BlockBlob`, which a SAS cannot carry; without it every uncached `COPY` in `Template.build()` failed with a storage `400 MissingRequiredHeader`. GCS and S3 clusters return no headers and are unaffected. diff --git a/packages/js-sdk/src/template/buildApi.ts b/packages/js-sdk/src/template/buildApi.ts index 489445c4ee..0b8f67ac9b 100644 --- a/packages/js-sdk/src/template/buildApi.ts +++ b/packages/js-sdk/src/template/buildApi.ts @@ -113,6 +113,7 @@ export async function uploadFile( fileName: string fileContextPath: string url: string + headers?: Record ignorePatterns: string[] resolveSymlinks: boolean gzip: boolean @@ -128,6 +129,7 @@ export async function uploadFile( const { fileName, url, + headers, fileContextPath, ignorePatterns, resolveSymlinks, @@ -154,7 +156,7 @@ export async function uploadFile( abortOpts?.signal ) - const res = await putFileStream(url, tar.path, tar.size, signal) + const res = await putFileStream(url, tar.path, tar.size, signal, headers) if (!res.ok) { throw new FileUploadError( @@ -166,17 +168,37 @@ export async function uploadFile( if (error instanceof FileUploadError) { throw error } - throw new FileUploadError(`Failed to upload file: ${error}`, stackTrace) + // fetch reports transport failures as a bare "TypeError: fetch failed"; + // the actual reason is on `cause`. + const cause = (error as { cause?: { message?: string } } | null)?.cause + ?.message + throw new FileUploadError( + `Failed to upload file: ${error}${cause ? ` (${cause})` : ''}`, + stackTrace + ) } finally { await cleanup?.() } } +const FRAMING_HEADERS = new Set(['content-length', 'transfer-encoding']) + +// The API's upload-header map is an open string map; a framing header from it +// would shadow or duplicate the archive's own Content-Length. +function withoutFramingHeaders(headers?: Record) { + return Object.fromEntries( + Object.entries(headers ?? {}).filter( + ([name]) => !FRAMING_HEADERS.has(name.toLowerCase()) + ) + ) +} + async function putFileStream( url: string, filePath: string, size: number, - signal: AbortSignal | undefined + signal: AbortSignal | undefined, + headers?: Record ): Promise<{ ok: boolean; statusText: string }> { // Prefer undici's fetch: it honors the explicit Content-Length on stream // bodies on every runtime, while Deno's native fetch ignores the header and @@ -192,7 +214,10 @@ async function putFileStream( body: stream.Readable.toWeb( fs.createReadStream(filePath) ) as ReadableStream, + // The upload link may require headers a signed URL cannot carry (Azure's + // Put Blob needs x-ms-blob-type); Content-Length is framing and wins. headers: { + ...withoutFramingHeaders(headers), 'Content-Length': size.toString(), }, // Streaming request bodies require half-duplex mode. diff --git a/packages/js-sdk/src/template/index.ts b/packages/js-sdk/src/template/index.ts index 1cc6401b28..12ba6caeb6 100644 --- a/packages/js-sdk/src/template/index.ts +++ b/packages/js-sdk/src/template/index.ts @@ -1112,7 +1112,7 @@ export class TemplateBase stackTrace = this.stackTraces[index + 1] } - const { present, url } = await getFileUploadLink( + const { present, url, headers } = await getFileUploadLink( client, { templateID, @@ -1131,6 +1131,7 @@ export class TemplateBase fileName: src, fileContextPath: this.fileContextPath.toString(), url, + headers, ignorePatterns: [ ...this.fileIgnorePatterns, ...readDockerignore(this.fileContextPath.toString()), diff --git a/packages/js-sdk/tests/template/uploadFile.test.ts b/packages/js-sdk/tests/template/uploadFile.test.ts index e9f7492c9c..d2044c0edb 100644 --- a/packages/js-sdk/tests/template/uploadFile.test.ts +++ b/packages/js-sdk/tests/template/uploadFile.test.ts @@ -18,6 +18,7 @@ describe('uploadFile transfer encoding', () => { let server: Server let baseUrl: string let capturedHeaders: IncomingMessage['headers'] = {} + let capturedHeaderNames: string[] = [] let capturedBodyLength = 0 beforeAll(async () => { @@ -26,6 +27,9 @@ describe('uploadFile transfer encoding', () => { server = createServer((req, res) => { capturedHeaders = req.headers + capturedHeaderNames = req.rawHeaders + .filter((_, i) => i % 2 === 0) + .map((name) => name.toLowerCase()) let bytes = 0 req.on('data', (chunk: Buffer) => { bytes += chunk.length @@ -73,5 +77,54 @@ describe('uploadFile transfer encoding', () => { // Content-Type (e.g. inferred from the archive's file extension) makes // the storage backend reject the upload with 403 Forbidden. expect(capturedHeaders['content-type']).toBeUndefined() + + expect(capturedHeaders['x-ms-blob-type']).toBeUndefined() + }) + + test('sends the headers the upload link requires alongside Content-Length', async () => { + await uploadFile( + { + fileName: '*.txt', + fileContextPath: testDir, + url: baseUrl, + headers: { 'x-ms-blob-type': 'BlockBlob' }, + ignorePatterns: [], + resolveSymlinks: false, + gzip: true, + }, + undefined + ) + + expect(capturedHeaders['x-ms-blob-type']).toBe('BlockBlob') + expectIntactFraming() }) + + test('drops a framing header the upload link returned', async () => { + await uploadFile( + { + fileName: '*.txt', + fileContextPath: testDir, + url: baseUrl, + headers: { 'x-ms-blob-type': 'BlockBlob', 'content-length': '1' }, + ignorePatterns: [], + resolveSymlinks: false, + gzip: true, + }, + undefined + ) + + expect(capturedHeaders['x-ms-blob-type']).toBe('BlockBlob') + expectIntactFraming() + }) + + function expectIntactFraming() { + expect(Number(capturedHeaders['content-length'])).toBe(capturedBodyLength) + expect( + capturedHeaderNames.filter((name) => name === 'content-length') + ).toHaveLength(1) + expect( + (capturedHeaders['transfer-encoding'] ?? '').toLowerCase() + ).not.toContain('chunked') + expect(capturedHeaders['content-type']).toBeUndefined() + } }) diff --git a/packages/js-sdk/tests/template/uploadHeaders.test.ts b/packages/js-sdk/tests/template/uploadHeaders.test.ts new file mode 100644 index 0000000000..290c5173e2 --- /dev/null +++ b/packages/js-sdk/tests/template/uploadHeaders.test.ts @@ -0,0 +1,118 @@ +import { afterAll, afterEach, beforeAll, expect, test } from 'vitest' +import { writeFile, mkdtemp, rm } from 'fs/promises' +import { join } from 'path' +import { tmpdir } from 'os' +import { createServer, type IncomingMessage, type Server } from 'http' +import { AddressInfo } from 'net' +import { http, HttpResponse } from 'msw' +import { setupServer } from 'msw/node' +import { randomUUID } from 'node:crypto' + +import { Template } from '../../src' +import { TEST_API_KEY, apiUrl } from '../setup' + +// The file-upload-link response carries headers the signed URL cannot carry +// itself (Azure's Put Blob requires x-ms-blob-type); without them every +// uncached COPY fails with a storage 400 on an Azure-backed cluster. + +let testDir: string +let uploadServer: Server +let uploadUrl: string +let capturedHeaders: IncomingMessage['headers'] = {} + +let linkHeaders: Record | undefined + +const restHandlers = [ + http.post(apiUrl('/v3/templates'), async ({ request }) => { + const { name } = (await request.clone().json()) as { name: string } + return HttpResponse.json({ + buildID: randomUUID(), + templateID: name, + tags: [], + }) + }), + http.get(apiUrl('/templates/:templateID/files/:hash'), () => + HttpResponse.json({ + present: false, + url: uploadUrl, + headers: linkHeaders, + }) + ), + http.post(apiUrl('/v2/templates/:templateID/builds/:buildID'), () => + HttpResponse.json({}) + ), +] + +const server = setupServer(...restHandlers) + +beforeAll(async () => { + testDir = await mkdtemp(join(tmpdir(), 'uploadHeaders-test-')) + await writeFile(join(testDir, 'hello.txt'), 'hello world') + + uploadServer = createServer((req, res) => { + capturedHeaders = req.headers + req.on('data', () => {}) + req.on('end', () => { + res.writeHead(200) + res.end() + }) + }) + await new Promise((resolve) => + uploadServer.listen(0, '127.0.0.1', resolve) + ) + const { port } = uploadServer.address() as AddressInfo + uploadUrl = `http://127.0.0.1:${port}/upload` + + // Only the local upload server may go unmocked. print.error() alone still + // performs the request, so anything else is blocked by throwing. + server.listen({ + onUnhandledRequest: (request, print) => { + if (new URL(request.url).hostname === '127.0.0.1') return + print.error() + throw new Error(`unhandled request: ${request.method} ${request.url}`) + }, + }) +}) + +afterAll(async () => { + server.close() + await new Promise((resolve) => uploadServer.close(() => resolve())) + await rm(testDir, { recursive: true, force: true }) +}) + +afterEach(() => { + capturedHeaders = {} + linkHeaders = undefined +}) + +test('upload PUT carries the headers the upload link returned', async () => { + linkHeaders = { 'x-ms-blob-type': 'BlockBlob' } + + const template = Template({ fileContextPath: testDir }) + .fromBaseImage() + .copy('*.txt', '.') + + await Template.buildInBackground(template, 'upload-headers', { + apiKey: TEST_API_KEY, + }) + + expect(capturedHeaders['x-ms-blob-type']).toBe('BlockBlob') + expect(Number(capturedHeaders['content-length'])).toBeGreaterThan(0) + expect( + (capturedHeaders['transfer-encoding'] ?? '').toLowerCase() + ).not.toContain('chunked') + expect(capturedHeaders['content-type']).toBeUndefined() +}) + +test('upload PUT is unchanged when the upload link returns no headers', async () => { + const template = Template({ fileContextPath: testDir }) + .fromBaseImage() + .copy('*.txt', '.') + + await Template.buildInBackground(template, 'upload-no-headers', { + apiKey: TEST_API_KEY, + }) + + expect(capturedHeaders['x-ms-blob-type']).toBeUndefined() + expect(Number(capturedHeaders['content-length'])).toBeGreaterThan(0) +}) From 42102f39df79d196752fe4760398f21a980d9b95 Mon Sep 17 00:00:00 2001 From: Tomas Srnka Date: Mon, 14 Sep 2026 18:22:35 +0000 Subject: [PATCH 2/2] fix(python-sdk): send the upload-request headers the API returns MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The file-upload-link response carries `headers` — request headers the upload must send that the signed URL cannot carry itself. The API sets them when the storage backend needs them: Azure signs layer-file uploads with a SAS and returns `x-ms-blob-type: BlockBlob`, which a SAS cannot express. Both builders passed only `file_info.url` to `upload_file`, whose signature had no headers parameter — the sync path PUT with no headers, the async path with `Content-Length` only. On an Azure-backed cluster every uncached `COPY` failed with a storage `400 MissingRequiredHeader`. GCS and S3 return no headers, which is why the live-API suites never saw it. Thread the headers into the PUT; the generated field is `UNSET` or a `TemplateBuildFileUploadHeaders`, and both a missing and an empty map behave exactly as before. The map is an open `additionalProperties: string`, so merging it verbatim breaks body framing. httpx applies its fstat-derived `Content-Length` with a case-insensitive `setdefault`, so an API `content-length: 1` wins on the sync path and truncates the archive to one byte with a 200 back; on the async path exact-key dedupe lets it through as a second header, which hyper forwards as a duplicate plus chunked encoding — the e2b#1243 failure. `strip_framing_headers` drops `content-length` and `transfer-encoding` case-insensitively, and both paths now apply an explicit `Content-Length` from the archive's size last. Verification: `test_build_forwards_upload_link_headers` drives `_build` against a stubbed upload link and a local PUT server in both suites, parametrized over headers set and `UNSET`; `test_upload_file_sends_required_headers` covers the `build_api` boundary and `test_upload_file_drops_framing_headers_from_the_api` pins the precedence with an API `content-length: 1`. All fail without this change. --- packages/python-sdk/e2b/template/utils.py | 20 ++- .../e2b/template_async/build_api.py | 18 ++- .../python-sdk/e2b/template_async/main.py | 6 + .../python-sdk/e2b/template_sync/build_api.py | 31 +++- packages/python-sdk/e2b/template_sync/main.py | 6 + .../async/template_async/test_upload_file.py | 140 +++++++++++++++++ .../sync/template_sync/test_upload_file.py | 141 ++++++++++++++++++ 7 files changed, 350 insertions(+), 12 deletions(-) diff --git a/packages/python-sdk/e2b/template/utils.py b/packages/python-sdk/e2b/template/utils.py index b61101b307..94d61be729 100644 --- a/packages/python-sdk/e2b/template/utils.py +++ b/packages/python-sdk/e2b/template/utils.py @@ -8,7 +8,7 @@ import re import inspect from types import TracebackType, FrameType -from typing import IO, List, Optional, Union +from typing import IO, Dict, List, Optional, Union from e2b.exceptions import TemplateException from e2b.template.consts import BASE_STEP_NAME, FINALIZE_STEP_NAME @@ -304,6 +304,24 @@ def tar_file_stream( raise +_FRAMING_HEADERS = frozenset({"content-length", "transfer-encoding"}) + + +def strip_framing_headers(headers: Optional[Dict[str, str]]) -> Dict[str, str]: + """ + Drop framing headers from an API-supplied header map. + + The upload-link header map is an open string map; a framing header from it + would shadow or duplicate the archive's own Content-Length. + + :param headers: Headers returned by the file-upload-link response + :return: The headers with any framing entry removed + """ + return { + k: v for k, v in (headers or {}).items() if k.lower() not in _FRAMING_HEADERS + } + + def strip_ansi_escape_codes(text: str) -> str: """ Strip ANSI escape codes from a string. diff --git a/packages/python-sdk/e2b/template_async/build_api.py b/packages/python-sdk/e2b/template_async/build_api.py index 328a002f9a..69eafa158e 100644 --- a/packages/python-sdk/e2b/template_async/build_api.py +++ b/packages/python-sdk/e2b/template_async/build_api.py @@ -1,7 +1,7 @@ import asyncio import os from types import TracebackType -from typing import Callable, Optional, List, Union +from typing import Callable, Dict, Optional, List, Union import httpx from pyqwest import HTTPTransport @@ -42,7 +42,11 @@ TemplateTagInfo, ) from e2b.template.consts import FILE_UPLOAD_TIMEOUT_SECONDS -from e2b.template.utils import get_build_step_index, tar_file_stream +from e2b.template.utils import ( + get_build_step_index, + strip_framing_headers, + tar_file_stream, +) async def request_build( @@ -115,6 +119,7 @@ async def upload_file( resolve_symlinks: bool, gzip: bool, stack_trace: Optional[TracebackType], + headers: Optional[Dict[str, str]] = None, request_timeout: Optional[float] = None, ): # Uploading a large build-context archive can take far longer than the 60s @@ -155,11 +160,16 @@ async def upload_file( # Stream the archive from disk via an async iterator. The # explicit Content-Length suppresses chunked transfer # encoding, which S3 presigned URLs reject; reqwest keeps the - # Content-Length framing for the streamed body. + # Content-Length framing for the streamed body. The link may + # also require headers a SAS cannot carry (Azure's Put Blob + # needs x-ms-blob-type); Content-Length wins over them. response = await client.put( url, content=aiter_io_chunks(tar_file), - headers={"Content-Length": str(size)}, + headers={ + **strip_framing_headers(headers), + "Content-Length": str(size), + }, ) response.raise_for_status() finally: diff --git a/packages/python-sdk/e2b/template_async/main.py b/packages/python-sdk/e2b/template_async/main.py index 2247f8b0ae..700069878e 100644 --- a/packages/python-sdk/e2b/template_async/main.py +++ b/packages/python-sdk/e2b/template_async/main.py @@ -4,6 +4,7 @@ from typing_extensions import Unpack from e2b.api.client.client import AuthenticatedClient +from e2b.api.client.types import Unset from e2b.connection_config import ApiParams, ConnectionConfig from e2b.template.consts import GZIP, RESOLVE_SYMLINKS from e2b.template.logger import LogEntry, LogEntryEnd, LogEntryStart @@ -137,6 +138,11 @@ async def _build( resolve_symlinks, gzip, stack_trace, + headers=( + None + if isinstance(file_info.headers, Unset) + else file_info.headers.to_dict() + ), request_timeout=request_timeout, ) if on_build_logs: diff --git a/packages/python-sdk/e2b/template_sync/build_api.py b/packages/python-sdk/e2b/template_sync/build_api.py index 735dcf7970..65538dc693 100644 --- a/packages/python-sdk/e2b/template_sync/build_api.py +++ b/packages/python-sdk/e2b/template_sync/build_api.py @@ -1,6 +1,7 @@ +import os import time from types import TracebackType -from typing import Callable, Optional, List, Union +from typing import Callable, Dict, Optional, List, Union import httpx from pyqwest import SyncHTTPTransport @@ -40,7 +41,11 @@ TemplateTagInfo, ) from e2b.template.consts import FILE_UPLOAD_TIMEOUT_SECONDS -from e2b.template.utils import get_build_step_index, tar_file_stream +from e2b.template.utils import ( + get_build_step_index, + strip_framing_headers, + tar_file_stream, +) def request_build( @@ -113,6 +118,7 @@ def upload_file( resolve_symlinks: bool, gzip: bool, stack_trace: Optional[TracebackType], + headers: Optional[Dict[str, str]] = None, request_timeout: Optional[float] = None, ): # Uploading a large build-context archive can take far longer than the 60s @@ -128,6 +134,8 @@ def upload_file( file_name, context_path, ignore_patterns, resolve_symlinks, gzip ) try: + size = os.fstat(tar_file.fileno()).st_size + # Through the pyqwest adapter the upload timeout is a # whole-request deadline for the entire transfer, not a per-write # bound as with the httpx transport this replaced. @@ -148,11 +156,20 @@ def upload_file( ) ), ) as client: - # httpx streams the archive from disk in chunks and sets - # Content-Length from the file size—S3 presigned URLs reject - # chunked transfer encoding, and reqwest keeps the - # Content-Length framing for the streamed body. - response = client.put(url, content=tar_file) + # Stream the archive from disk under an explicit + # Content-Length: S3 presigned URLs reject chunked transfer + # encoding, and reqwest keeps the Content-Length framing for + # the streamed body. The link may also require headers a SAS + # cannot carry (Azure's Put Blob needs x-ms-blob-type); + # Content-Length wins over them. + response = client.put( + url, + content=tar_file, + headers={ + **strip_framing_headers(headers), + "Content-Length": str(size), + }, + ) response.raise_for_status() finally: # Closing the spooled temp file is best-effort: a failure here diff --git a/packages/python-sdk/e2b/template_sync/main.py b/packages/python-sdk/e2b/template_sync/main.py index 1481966e04..ad0d0f9247 100644 --- a/packages/python-sdk/e2b/template_sync/main.py +++ b/packages/python-sdk/e2b/template_sync/main.py @@ -4,6 +4,7 @@ from typing_extensions import Unpack from e2b.api.client.client import AuthenticatedClient +from e2b.api.client.types import Unset from e2b.connection_config import ApiParams, ConnectionConfig from e2b.api.client_sync import get_api_client @@ -137,6 +138,11 @@ def _build( resolve_symlinks, gzip, stack_trace, + headers=( + None + if isinstance(file_info.headers, Unset) + else file_info.headers.to_dict() + ), request_timeout=request_timeout, ) if on_build_logs: diff --git a/packages/python-sdk/tests/async/template_async/test_upload_file.py b/packages/python-sdk/tests/async/template_async/test_upload_file.py index a8d2993427..a9baf27f32 100644 --- a/packages/python-sdk/tests/async/template_async/test_upload_file.py +++ b/packages/python-sdk/tests/async/template_async/test_upload_file.py @@ -1,15 +1,24 @@ +import os import threading from http.server import BaseHTTPRequestHandler, HTTPServer +from types import SimpleNamespace from typing import Any, Dict from unittest import mock import httpx import pytest +from e2b import AsyncTemplate from e2b.api.client.client import AuthenticatedClient +from e2b.api.client.models import TemplateBuildFileUpload +from e2b.api.client.models.template_build_file_upload_headers import ( + TemplateBuildFileUploadHeaders, +) +from e2b.api.client.types import UNSET from e2b.template import utils as template_utils from e2b.exceptions import FileUploadException from e2b.template.consts import FILE_UPLOAD_TIMEOUT_SECONDS +import e2b.template_async.main as template_async_main from e2b.template_async.build_api import upload_file @@ -32,6 +41,7 @@ def do_PUT(self): # hyper (pyqwest) sends lowercase header names where httpcore # title-cased them; compare case-insensitively. state["headers"] = {k.lower(): v for k, v in self.headers.items()} + state["header_names"] = [k.lower() for k, _ in self.headers.items()] state["paths"].append(self.path) length = int(self.headers.get("Content-Length", 0)) body = self.rfile.read(length) if length else b"" @@ -240,3 +250,133 @@ def failing_close_stream(*args, **kwargs): thread.join(timeout=5) assert state["headers"] is not None + + +def _archive_size(tmp_path) -> int: + with template_utils.tar_file_stream("*.txt", str(tmp_path), [], False, True) as f: + return os.fstat(f.fileno()).st_size + + +def _assert_intact_framing(state, tmp_path): + headers = state["headers"] + assert int(headers["content-length"]) == _archive_size(tmp_path) + assert state["body_length"] == _archive_size(tmp_path) + assert state["header_names"].count("content-length") == 1 + assert "chunked" not in headers.get("transfer-encoding", "").lower() + assert "content-type" not in headers + + +# The file-upload-link response carries headers the signed URL cannot carry +# itself (Azure's Put Blob requires x-ms-blob-type); without them every +# uncached COPY fails with a storage 400 on an Azure-backed cluster. + + +async def test_upload_file_sends_required_headers(tmp_path): + (tmp_path / "hello.txt").write_text("hello world") + + server, thread, state = _make_server() + host, port = server.server_address + url = f"http://{host}:{port}/upload" + + try: + await upload_file( + api_client=AuthenticatedClient(base_url="http://test", token="test"), + file_name="*.txt", + context_path=str(tmp_path), + url=url, + ignore_patterns=[], + resolve_symlinks=False, + gzip=True, + stack_trace=None, + headers={"x-ms-blob-type": "BlockBlob"}, + ) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + assert state["headers"]["x-ms-blob-type"] == "BlockBlob" + _assert_intact_framing(state, tmp_path) + + +@pytest.mark.parametrize( + "link_headers, expected_blob_type", + [ + ( + TemplateBuildFileUploadHeaders.from_dict({"x-ms-blob-type": "BlockBlob"}), + "BlockBlob", + ), + (UNSET, None), + ], +) +async def test_build_forwards_upload_link_headers( + tmp_path, monkeypatch, link_headers, expected_blob_type +): + (tmp_path / "hello.txt").write_text("hello world") + + server, thread, state = _make_server() + host, port = server.server_address + url = f"http://{host}:{port}/upload" + + async def fake_request_build(*args, **kwargs): + return SimpleNamespace(template_id="template-id", build_id="build-id", tags=[]) + + async def fake_trigger_build(*args, **kwargs): + return None + + async def fake_get_file_upload_link(*args, **kwargs): + return TemplateBuildFileUpload(present=False, url=url, headers=link_headers) + + monkeypatch.setattr(template_async_main, "request_build", fake_request_build) + monkeypatch.setattr(template_async_main, "trigger_build", fake_trigger_build) + monkeypatch.setattr( + template_async_main, "get_file_upload_link", fake_get_file_upload_link + ) + + template = ( + AsyncTemplate(file_context_path=str(tmp_path)) + .from_base_image() + .copy("*.txt", ".") + ) + + try: + await AsyncTemplate._build( + AuthenticatedClient(base_url="http://test", token="test"), + template, + "upload-headers", + ) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + assert state["headers"].get("x-ms-blob-type") == expected_blob_type + _assert_intact_framing(state, tmp_path) + + +async def test_upload_file_drops_framing_headers_from_the_api(tmp_path): + (tmp_path / "hello.txt").write_text("hello world") + + server, thread, state = _make_server() + host, port = server.server_address + url = f"http://{host}:{port}/upload" + + try: + await upload_file( + api_client=AuthenticatedClient(base_url="http://test", token="test"), + file_name="*.txt", + context_path=str(tmp_path), + url=url, + ignore_patterns=[], + resolve_symlinks=False, + gzip=True, + stack_trace=None, + headers={"x-ms-blob-type": "BlockBlob", "content-length": "1"}, + ) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + assert state["headers"]["x-ms-blob-type"] == "BlockBlob" + _assert_intact_framing(state, tmp_path) diff --git a/packages/python-sdk/tests/sync/template_sync/test_upload_file.py b/packages/python-sdk/tests/sync/template_sync/test_upload_file.py index a08ae9906a..f342e80be3 100644 --- a/packages/python-sdk/tests/sync/template_sync/test_upload_file.py +++ b/packages/python-sdk/tests/sync/template_sync/test_upload_file.py @@ -1,15 +1,24 @@ +import os import threading from http.server import BaseHTTPRequestHandler, HTTPServer +from types import SimpleNamespace from typing import Any, Dict from unittest import mock import httpx import pytest +from e2b import Template from e2b.api.client.client import AuthenticatedClient +from e2b.api.client.models import TemplateBuildFileUpload +from e2b.api.client.models.template_build_file_upload_headers import ( + TemplateBuildFileUploadHeaders, +) +from e2b.api.client.types import UNSET from e2b.template import utils as template_utils from e2b.exceptions import FileUploadException from e2b.template.consts import FILE_UPLOAD_TIMEOUT_SECONDS +import e2b.template_sync.main as template_sync_main from e2b.template_sync.build_api import upload_file @@ -28,6 +37,7 @@ def do_PUT(self): # hyper (pyqwest) sends lowercase header names where httpcore # title-cased them; compare case-insensitively. state["headers"] = {k.lower(): v for k, v in self.headers.items()} + state["header_names"] = [k.lower() for k, _ in self.headers.items()] state["paths"].append(self.path) length = int(self.headers.get("Content-Length", 0)) body = self.rfile.read(length) if length else b"" @@ -236,3 +246,134 @@ def failing_close_stream(*args, **kwargs): thread.join(timeout=5) assert state["headers"] is not None + + +def _archive_size(tmp_path) -> int: + with template_utils.tar_file_stream("*.txt", str(tmp_path), [], False, True) as f: + return os.fstat(f.fileno()).st_size + + +def _assert_intact_framing(state, tmp_path): + headers = state["headers"] + assert int(headers["content-length"]) == _archive_size(tmp_path) + assert state["body_length"] == _archive_size(tmp_path) + assert state["header_names"].count("content-length") == 1 + assert "chunked" not in headers.get("transfer-encoding", "").lower() + assert "content-type" not in headers + + +# The file-upload-link response carries headers the signed URL cannot carry +# itself (Azure's Put Blob requires x-ms-blob-type); without them every +# uncached COPY fails with a storage 400 on an Azure-backed cluster. + + +def test_upload_file_sends_required_headers(tmp_path): + (tmp_path / "hello.txt").write_text("hello world") + + server, thread, state = _make_server() + host, port = server.server_address + url = f"http://{host}:{port}/upload" + + try: + upload_file( + api_client=AuthenticatedClient(base_url="http://test", token="test"), + file_name="*.txt", + context_path=str(tmp_path), + url=url, + ignore_patterns=[], + resolve_symlinks=False, + gzip=True, + stack_trace=None, + headers={"x-ms-blob-type": "BlockBlob"}, + ) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + assert state["headers"]["x-ms-blob-type"] == "BlockBlob" + _assert_intact_framing(state, tmp_path) + + +@pytest.mark.parametrize( + "link_headers, expected_blob_type", + [ + ( + TemplateBuildFileUploadHeaders.from_dict({"x-ms-blob-type": "BlockBlob"}), + "BlockBlob", + ), + (UNSET, None), + ], +) +def test_build_forwards_upload_link_headers( + tmp_path, monkeypatch, link_headers, expected_blob_type +): + (tmp_path / "hello.txt").write_text("hello world") + + server, thread, state = _make_server() + host, port = server.server_address + url = f"http://{host}:{port}/upload" + + monkeypatch.setattr( + template_sync_main, + "request_build", + lambda *args, **kwargs: SimpleNamespace( + template_id="template-id", build_id="build-id", tags=[] + ), + ) + monkeypatch.setattr( + template_sync_main, "trigger_build", lambda *args, **kwargs: None + ) + monkeypatch.setattr( + template_sync_main, + "get_file_upload_link", + lambda *args, **kwargs: TemplateBuildFileUpload( + present=False, url=url, headers=link_headers + ), + ) + + template = ( + Template(file_context_path=str(tmp_path)).from_base_image().copy("*.txt", ".") + ) + + try: + Template._build( + AuthenticatedClient(base_url="http://test", token="test"), + template, + "upload-headers", + ) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + assert state["headers"].get("x-ms-blob-type") == expected_blob_type + _assert_intact_framing(state, tmp_path) + + +def test_upload_file_drops_framing_headers_from_the_api(tmp_path): + (tmp_path / "hello.txt").write_text("hello world") + + server, thread, state = _make_server() + host, port = server.server_address + url = f"http://{host}:{port}/upload" + + try: + upload_file( + api_client=AuthenticatedClient(base_url="http://test", token="test"), + file_name="*.txt", + context_path=str(tmp_path), + url=url, + ignore_patterns=[], + resolve_symlinks=False, + gzip=True, + stack_trace=None, + headers={"x-ms-blob-type": "BlockBlob", "content-length": "1"}, + ) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + assert state["headers"]["x-ms-blob-type"] == "BlockBlob" + _assert_intact_framing(state, tmp_path)