Feature/minimalist thriller pipeline

README.md

@@ -14,6 +14,8 @@ ImgGenHub is a personal image generation hub that connects to web-based image ge
 
 #### **Kaggle-powered image generation**
 - **Automated pipeline**: Deploy → Monitor → Download workflow
+- **Parallel Execution**: Automatically splits large prompt batches across multiple GPUs.
+- **Descriptive Naming**: Generated images include a slugified version of the prompt for easy identification.
 - **Automated secret management**: Local `.env` secret detected and auto-uploaded to Kaggle dataset
 - **Multiple models**: Supports Stable Diffusion variants, Flux.1-schnell GGUF quantized (Q4) version and Flux.1-schnell bf16 version.
 
@@ -48,23 +50,27 @@ poetry run imggenhub \
   --img_width 1024 \
   --img_height 1024 \
   --precision bf16 \
-  --gpu
+  --gpu t4x2
 ```
 
 ### Multiple prompts via command line
 
+For larger batches (>4 prompts), ImgGenHub automatically splits the workload into two parallel kernels.
+
 ```bash
 poetry run imggenhub \
   --prompt "A serene mountain landscape at sunset" \
   --prompt "A bustling city street in the rain" \
   --prompt "An abstract geometric pattern" \
+  --prompt "A neon cyberpunk cityscape" \
+  --prompt "A futuristic laboratory" \
   --model_id "black-forest-labs/FLUX.1-schnell" \
   --steps 4 \
   --guidance 0.75 \
   --img_width 1024 \
   --img_height 1024 \
   --precision bf16 \
-  --gpu
+  --gpu p100
 ```
 
 ### FLUX.1-schnell quantized version (Q4_0 GGUF)
@@ -75,13 +81,59 @@ poetry run imggenhub \
   --model_id "city96/FLUX.1-schnell-gguf" \
   --model_filename "flux1-schnell-Q4_0.gguf" \
   --steps 4 \
-  --guidance 0.8 \
-  --img_width 512 \
-  --img_height 512 \
+  --guidance 1.0 \
+  --img_width 1024 \
+  --img_height 1024 \
+  --precision q4 \
+  --gpu
+```
+
+### Multiple prompts via JSON file (Parallel Mode)
+
+For larger batches, ImgGenHub automatically splits prompts across multiple Kaggle kernels to stay within runtime limits and speed up generation.
+
+```bash
+poetry run imggenhub \
+  --prompts_file path/to/prompts.json \
+  --model_id "city96/FLUX.1-schnell-gguf" \
+  --model_filename "flux1-schnell-Q4_0.gguf" \
+  --steps 4 \
+  --guidance 1.0 \
+  --img_width 1024 \
+  --img_height 1024 \
   --precision q4 \
   --gpu
 ```
 
+### 🎬 Storyteller Example (Parallel Mode)
+
+For cinematic storyboards or large batches, ImgGenHub splits prompts across multiple parallel Kaggle kernels to stay within runtime limits.
+
+<details>
+<summary><b>Example: 12-Beat Thriller Pipeline</b></summary>
+
+1. **Save prompts to `thriller.json`**:
+```json
+[
+  "minimalist black and white photo of a dark alley",
+  "minimalist red high heel shoe on a puddle",
+  "close up of a neon blinking open sign",
+  ...
+]
+```
+
+2. **Run across 2 parallel GPUs (T4 x2)**:
+```bash
+poetry run imggenhub --prompts_file thriller.json \
+  --model_id city96/FLUX.1-schnell-gguf \
+  --model_filename flux1-schnell-Q4_0.gguf \
+  --parallel_mode kaggle-t4-x2 \
+  --img_width 1024 --img_height 576 --steps 4 --gpu
+```
+
+_Outputs will be saved with descriptive names like `gen_p1_minimalist_black_and_white_pho...png`_
+</details>
+
 ### Stable diffusion XL with refiner 
 
 ```bash
@@ -101,9 +153,9 @@ poetry run imggenhub \
 ### **Supported flags**
 
 #### **General flags** (all models)
-- `--prompt`: Single prompt or multiple prompts (use flag multiple times)
-- `--prompts_file`: JSON file with multiple prompts  
-- `--gpu`: Enable GPU acceleration (required for FLUX.1 models)
+- `--prompt`: Single prompt or multiple prompts (use flag multiple times). Automatically enables parallel mode if >4 prompts are submitted.
+- `--prompts_file`: JSON file with multiple prompts. Automatically enables parallel mode if >4 prompts are submitted.
+- `--gpu [type]`: Enable GPU acceleration (required for FLUX.1 models). Optional type: `t4x2` (default) or `p100`.
 - `--steps`: Inference steps (50-100 for Stable Diffusion models, ~4 for FLUX)
 - `--guidance`: Prompt adherence strength (7-12 recommended for photorealism, 0.75-1.0 for FLUX)
 - `--precision`: Model precision (fp32/fp16/int8/int4 for base models; q4/q5/q6 for GGUF quantized models)

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "imggenhub"
-version = "1.8.0"
+version = "1.8.1"
 description = ""
 authors = ["leweex95 <csibi.levente14@gmail.com>"]
 readme = "README.md"

src/imggenhub/kaggle/core/parallel_deploy.py

@@ -56,7 +56,8 @@ def _deploy_single_kernel(
     kernel_path: Path,
     kernel_id: str,
     deploy_kwargs: Dict[str, Any],
-    accelerator: str = None
+    accelerator: str = None,
+    index_offset: int = 0
 ) -> str:
     """
     Deploy a single kernel with given prompts using Kaggle Connector.
@@ -68,6 +69,7 @@ def _deploy_single_kernel(
         kernel_id: Kernel identifier
         deploy_kwargs: Additional kwargs for JobManager
         accelerator: Kaggle accelerator type
+        index_offset: Offset for image numbering
 
     Returns:
         Kernel ID that was deployed
@@ -89,22 +91,31 @@ def _deploy_single_kernel(
         # Prepare parameters for injection
         params = {
             "PROMPTS": prompts_list,
-            "MODEL_ID": deploy_kwargs.get("model_id"),
-            "GUIDANCE": deploy_kwargs.get("guidance"),
-            "STEPS": deploy_kwargs.get("steps"),
-            "PRECISION": deploy_kwargs.get("precision"),
             "OUTPUT_DIR": ".",
-            "IMG_SIZE": deploy_kwargs.get("img_size"),
-            "KERNEL_ID": kernel_id
+            "KERNEL_ID": kernel_id,
+            "INDEX_OFFSET": index_offset
         }
 
-        # Flux GGUF specific
-        if "model_filename" in deploy_kwargs:
-            params["MODEL_FILENAME"] = deploy_kwargs.get("model_filename")
-        if "vae_repo_id" in deploy_kwargs:
-            params["VAE_REPO_ID"] = deploy_kwargs.get("vae_repo_id")
-        if "vae_filename" in deploy_kwargs:
-            params["VAE_FILENAME"] = deploy_kwargs.get("vae_filename")
+        # Only add optional params if they are provided to avoid overwriting notebook defaults with None
+        optional_params = [
+            ("MODEL_ID", "model_id"),
+            ("GUIDANCE", "guidance"),
+            ("STEPS", "steps"),
+            ("PRECISION", "precision"),
+            ("IMG_SIZE", "img_size"),
+            ("MODEL_FILENAME", "model_filename"),
+            ("VAE_REPO_ID", "vae_repo_id"),
+            ("VAE_FILENAME", "vae_filename"),
+            ("CLIP_L_REPO_ID", "clip_l_repo_id"),
+            ("CLIP_L_FILENAME", "clip_l_filename"),
+            ("T5XXL_REPO_ID", "t5xxl_repo_id"),
+            ("T5XXL_FILENAME", "t5xxl_filename"),
+        ]
+
+        for nb_key, kw_key in optional_params:
+            val = deploy_kwargs.get(kw_key)
+            if val is not None:
+                params[nb_key] = val
 
         manager.edit_notebook_params(str(tmp_nb_path), params)
 
@@ -236,28 +247,30 @@ def run_parallel_pipeline(
         logging.info("PARALLEL DEPLOYMENT: Deploying to 2 Kaggle kernels")
         logging.info("="*80)
 
-        # Deploy deployment1 kernel first
+        # Deploy deployment1 kernel first (no offset)
         _deploy_single_kernel(
             prompts_list=first_batch,
             notebook=notebook,
             kernel_path=kernel_path,
             kernel_id=deployment1_kernel_id,
             deploy_kwargs={**deploy_kwargs, "wait_timeout": wait_timeout, "retry_interval": retry_interval},
-            accelerator=accelerator
+            accelerator=accelerator,
+            index_offset=0
         )
 
         # Wait before deploying deployment2 to avoid API conflicts
         logging.info("Waiting 15 seconds before deploying deployment2 kernel...")
         time.sleep(15)
 
-        # Deploy deployment2 kernel
+        # Deploy deployment2 kernel (with offset = size of first batch)
         _deploy_single_kernel(
             prompts_list=second_batch,
             notebook=notebook,
             kernel_path=kernel_path,
             kernel_id=deployment2_kernel_id,
             deploy_kwargs={**deploy_kwargs, "wait_timeout": wait_timeout, "retry_interval": retry_interval},
-            accelerator=accelerator
+            accelerator=accelerator,
+            index_offset=len(first_batch)
         )
 
         logging.info("="*80)
@@ -295,8 +308,15 @@ def run_parallel_pipeline(
         if errors:
             logging.error(f"The following kernels failed: {errors}")
             # Identify first error message
-            first_err = next((status for kid, status in statuses.items() if kid in errors), "Unknown error")
-            raise RuntimeError(f"Parallel kernel execution failed: {first_err}")
+            err_kid = errors[0]
+            manager = JobManager(err_kid)
+            try:
+                logs = manager.get_logs()
+                msg = logs.strip() if logs else f"no logs available (status: {statuses[err_kid]})"
+            except Exception as e:
+                msg = f"could not retrieve logs: {e} (status: {statuses[err_kid]})"
+
+            raise RuntimeError(f"Parallel kernel execution failed: {err_kid}\n{msg}")
 
         logging.info("="*80)
         logging.info("Both kernels completed! Downloading outputs...")
@@ -343,8 +363,10 @@ def run_parallel_pipeline(
         for temp_path in [deployment1_download_path, deployment2_download_path]:
             if not temp_path.exists(): continue
             for image_file in temp_path.rglob("*"):
-                # ONLY collect files that start with 'gen_' to avoid pulling stale artifacts
-                if image_file.is_file() and image_file.suffix.lower() in image_extensions and image_file.name.startswith("gen_"):
+                # ONLY collect files that start with 'gen_' or 'generated_' to avoid pulling stale artifacts
+                is_image = image_file.is_file() and image_file.suffix.lower() in image_extensions
+                is_valid_prefix = image_file.name.startswith("gen_") or image_file.name.startswith("generated_")
+                if is_image and is_valid_prefix:
                     # Deduplicate by filename
                     if image_file.name not in unique_images:
                         unique_images[image_file.name] = image_file