Skip to content
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 60 additions & 0 deletions @blaxel/core/src/sandbox/filesystem/filesystem.ts
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,12 @@ export class SandboxFileSystem extends SandboxAction {
if (!fs) {
throw new Error("File path upload is only supported in Node.js environments");
}
const stat = fs.statSync(content);
// For files larger than MULTIPART_THRESHOLD, stream chunks directly
// to avoid hitting Node.js Buffer size limit (2 GiB)
if (stat.size > MULTIPART_THRESHOLD) {
return await this.uploadFileWithMultipart(path, content, stat.size, "0644");
}
const buffer = fs.readFileSync(content);
fileBlob = new Blob([buffer]);
} else {
Expand Down Expand Up @@ -476,6 +482,60 @@ export class SandboxFileSystem extends SandboxAction {
return data;
}

private async uploadFileWithMultipart(path: string, filePath: string, fileSize: number, permissions: string = "0644"): Promise<SuccessResponse> {
if (!fs) {
throw new Error("File streaming upload is only supported in Node.js environments");
}

const initResponse = await this.initiateMultipartUpload(path, permissions);
const uploadId = initResponse.uploadId;

if (!uploadId) {
throw new Error("Failed to get upload ID from initiate response");
}

try {
const numParts = Math.ceil(fileSize / CHUNK_SIZE);
const parts: Array<MultipartPartInfo> = [];
const fd = fs.openSync(filePath, 'r');

try {
for (let i = 0; i < numParts; i += MAX_PARALLEL_UPLOADS) {
const batch: Array<Promise<MultipartUploadPartResponse>> = [];

for (let j = 0; j < MAX_PARALLEL_UPLOADS && i + j < numParts; j++) {
const partNumber = i + j + 1;
const start = (partNumber - 1) * CHUNK_SIZE;
const end = Math.min(start + CHUNK_SIZE, fileSize);
const length = end - start;

const buffer = Buffer.alloc(length);
fs.readSync(fd, buffer, 0, length, start);
const chunk = new Blob([buffer]);

batch.push(this.uploadPart(uploadId, partNumber, chunk));
}

const batchResults = await Promise.all(batch);
parts.push(...batchResults);
}
} finally {
fs.closeSync(fd);
}

parts.sort((a, b) => (a.partNumber ?? 0) - (b.partNumber ?? 0));

return await this.completeMultipartUpload(uploadId, parts);
} catch (error) {
try {
await this.abortMultipartUpload(uploadId);
} catch (abortError) {
// Log but don't throw - we want to throw the original error
}
throw error;
}
}

private async uploadWithMultipart(path: string, blob: Blob, permissions: string = "0644"): Promise<SuccessResponse> {
// Initiate multipart upload

Expand Down
Loading