Local Telegram Bot API + streaming chunked transfers

.env.example

@@ -6,7 +6,25 @@ SUPERUSER_PASS=pentaract
 ACCESS_TOKEN_EXPIRE_IN_SECS=1800
 REFRESH_TOKEN_EXPIRE_IN_DAYS=14
 SECRET_KEY=XXX
-TELEGRAM_API_BASE_URL=https://api.telegram.org
+
+# --- Telegram integration ---
+# If TELEGRAM_LOCAL_API=true, the server will talk to the Local Telegram Bot API
+# (telegram-bot-api container) and can upload/download much larger chunks.
+TELEGRAM_LOCAL_API=true
+TELEGRAM_API_BASE_URL=http://telegram-bot-api:8081
+
+# Requests per minute per bot (scheduler enforces this via DB)
+TELEGRAM_RATE_LIMIT=18
+
+# Chunk size (MB). For local Bot API you can go up to ~1950.
+TELEGRAM_CHUNK_SIZE_MB=1950
+
+# Where Pentaract spools uploads before sending to Telegram
+WORK_DIR=/work
+
+# Needed ONLY for the telegram-bot-api container
+TELEGRAM_API_ID=00000000
+TELEGRAM_API_HASH=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
 
 DATABASE_USER=pentaract
 DATABASE_PASSWORD=pentaract

Dockerfile

@@ -10,7 +10,11 @@ RUN cargo install cargo-chef
 WORKDIR /app
 
 FROM chef AS planner
-COPY ./pentaract .
+WORKDIR /app
+COPY pentaract/Cargo.toml .
+COPY pentaract/Cargo.lock .
+COPY pentaract/src ./src
+
 RUN cargo chef prepare --recipe-path recipe.json
 
 FROM chef AS builder 

README.md

@@ -260,6 +260,21 @@ Currently Telegram API limits file download to 20 MB, hence we can't upload file
 
 Pentaract divides uploaded files into chunks and save them to Telegram separately and on downloading a file it fetches all the file chunks from the Telegram API and combine them into one in the order it was divided in. That grants ability to upload and download files with almost unlimited size (it's like you've ever downloaded a file with size >10 GB).
 
+#### Using Local Telegram Bot API (recommended for large files)
+
+If you run Telegram's **Local Bot API server** (aka `telegram-bot-api` with `--local`), Telegram allows much larger uploads (up to ~2GB per document).
+
+This repo now supports running it automatically via `docker-compose.yml`.
+
+Set these variables in `.env`:
+
+- `TELEGRAM_LOCAL_API=true`
+- `TELEGRAM_API_BASE_URL=http://telegram-bot-api:8081`
+- `TELEGRAM_API_ID` and `TELEGRAM_API_HASH` (your Telegram app credentials)
+- `TELEGRAM_CHUNK_SIZE_MB=1950` (safe default under the 2GB limit)
+
+With Local Bot API enabled, Pentaract will chunk and stream uploads/downloads so big files do not require loading everything into RAM.
+
 ## Current in storage features
 
 - [x] Upload file

docker-compose.yml

@@ -1,8 +1,10 @@
-version: "3.9"
-
 volumes:
   pentaract-db-volume:
     name: pentaract-db-volume
+  pentaract-work-volume:
+    name: pentaract-work-volume
+  telegram-bot-api-data:
+    name: telegram-bot-api-data
 
 networks:
   pentaract-network:
@@ -20,6 +22,25 @@ services:
     restart: unless-stopped
     depends_on:
       - db
+      - telegram-bot-api
+    volumes:
+      - pentaract-work-volume:/work
+      - telegram-bot-api-data:/var/lib/telegram-bot-api:ro
+    networks:
+      - pentaract-network
+
+  telegram-bot-api:
+    container_name: telegram-bot-api
+    image: aiogram/telegram-bot-api:latest
+    environment:
+      TELEGRAM_API_ID: ${TELEGRAM_API_ID}
+      TELEGRAM_API_HASH: ${TELEGRAM_API_HASH}
+      TELEGRAM_LOCAL: 1
+    restart: unless-stopped
+    ports:
+      - ${TELEGRAM_BOT_API_PORT:-8081}:8081
+    volumes:
+      - telegram-bot-api-data:/var/lib/telegram-bot-api
     networks:
       - pentaract-network
 

pentaract/Cargo.toml

@@ -3,35 +3,35 @@ name = "pentaract"
 version = "0.1.0"
 edition = "2021"
 
-[profile.release]
-strip = true
-codegen-units = 1
-
 [dependencies]
 # routing
-axum = { version = "0.6.20", features = ["headers", "tracing", "multipart"]}
+axum = { version = "0.6.20", features = ["headers", "tracing", "multipart"] }
 mime_guess = "2.0.4"
-tower = { version = "0.4.13", features = ["limit"], default-features = false}
-tower-http = { version = "0.4.4", features = ["fs", "trace", "cors"], default_features = false }
+tower = { version = "0.4.13", features = ["limit"], default-features = false }
+tower-http = { version = "0.4.4", features = ["fs", "trace", "cors"], default-features = false }
 
 # serialization/deserialization
 serde = { version = "1.0.189", features = ["derive"] }
 
 # auth
 pwhash = "1.0.0"
-jsonwebtoken = { version = "9", default-features = false }
+jsonwebtoken = { version = "9", default-features = false, features = ["use_pem"] }
 
 # async
 tokio = { version = "1.33.0", features = ["full"] }
-tokio-util = "0.7.10"
+tokio-util = { version = "0.7.10", features = ["io"] }
 futures = "0.3.29"
+async-stream = "0.3.5"
 
 # logging
 tracing = "0.1.40"
-tracing-subscriber = { version = "0.3.17", features = ["env-filter"]} 
+tracing-subscriber = { version = "0.3.17", features = ["env-filter"] }
 
 # others
 sqlx = { version = "0.7", features = ["runtime-tokio-rustls", "postgres", "uuid"] }
 thiserror = "1.0.50"
 uuid = { version = "1.5.0", features = ["serde", "v4"] }
-reqwest = { version = "0.11.22", features = ["multipart", "json"] }
+
+# HTTP client (без native-tls — ок для musl)
+reqwest = { version = "0.11.22", default-features = false, features = ["multipart", "json", "stream", "rustls-tls"] }
+percent-encoding = "2.3"

pentaract/src/common/channels.rs

@@ -1,4 +1,5 @@
 use tokio::sync::{mpsc, oneshot};
+use std::path::PathBuf;
 use uuid::Uuid;
 
 use crate::errors::PentaractResult;
@@ -20,7 +21,8 @@ pub enum ClientData {
 pub struct UploadFileData {
     pub file_id: Uuid,
     pub user_id: Uuid,
-    pub file_data: Box<[u8]>,
+    pub file_path: PathBuf,
+    pub file_size: i64,
 }
 
 pub struct DownloadFileData {