Problems in Reproducing FLUX’s Shape and Texture Property

[HHHHStar]

Hi, thanks for your excellent benchmark!
When reproducing the results of shape and tuxture, I met significant mismatch. Would you mind sharing the inference_eval.py file for FLUX?

[Karine-Huang]

Hello, here is the script for generating the images:

import torch
from diffusers import FluxPipeline

import argparse
import os
from tqdm import tqdm, trange
from PIL import Image

def image_grid(imgs, rows, cols):
    assert len(imgs) == rows * cols
    w, h = imgs[0].size
    grid = Image.new('RGB', size=(cols * w, rows * h))
    grid_w, grid_h = grid.size
    for i, img in enumerate(imgs):
        grid.paste(img, box=(i % cols * w, i // cols * h))
    return grid

parser = argparse.ArgumentParser()
parser.add_argument(
    "--from-file",
    type=str,
    default="color_val.txt",
    help="if specified, load prompts from this file",
)
parser.add_argument(
    "--n_iter",
    type=int,
    default=10,
    help="sample this often",
)
parser.add_argument(
    "--batch_size",
    type=int,
    default=1, 
    help="how many samples to produce for each given prompt. A.k.a. batch size",
)
parser.add_argument(
    "--prompt",
    type=str,
    default="A girl on the left of a horse",
)
parser.add_argument(
    "--outdir",
    type=str,
    nargs="?",
    help="dir to write results to",
    default="./flux"
)
parser.add_argument(
    "--i",
    type=int,
    default=0,
    help="num to generate images",
)

opt = parser.parse_args()

if not opt.from_file:
    prompt = opt.prompt
    assert prompt is not None
    data = [opt.batch_size * [prompt]]

else:
    print(f"reading prompts from {opt.from_file}")
    with open(opt.from_file, "r") as f:
        data = f.read().splitlines()
        data = [d.strip("\n").split("\t")[0] for d in data]
        data = [opt.batch_size * [d] for d in data]

images = []
outpath = opt.outdir
os.makedirs(outpath, exist_ok=True)
sample_path = os.path.join(outpath, f"samples")
os.makedirs(sample_path, exist_ok=True)
base_count = len(os.listdir(sample_path))
grid_count = len(os.listdir(outpath)) - 1

model_id = "black-forest-labs/FLUX.1-schnell" 

pipe = FluxPipeline.from_pretrained("black-forest-labs/FLUX.1-schnell", torch_dtype=torch.bfloat16)
pipe.enable_sequential_cpu_offload()

with torch.no_grad():
    for prompts in tqdm(data, desc="data"): # TODO
        images = []
        generator = torch.Generator(device="cuda").manual_seed(42)
        for n in trange(opt.n_iter, desc="Sampling"):
            image = pipe(
                prompts,
                output_type="pil",
                num_inference_steps=4, 
                generator=generator,
            ).images
            generator = torch.Generator(device="cuda").manual_seed(42 + n + 1)
            for i in range(len(image)):
                image[i].save(os.path.join(sample_path, f"{prompts[0]}_{base_count:06}.png"))
                images.append(image[i])
                base_count += 1
        grid = image_grid(images, rows=opt.n_iter, cols=opt.batch_size)
        grid.save(os.path.join(outpath, f'{prompts[0]}-grid-{grid_count:05}.png'))
        grid_count += 1

Here is the env we use:

name: flux
channels:
  - defaults
  - conda-forge
dependencies:
  - _libgcc_mutex=0.1=main
  - _openmp_mutex=5.1=1_gnu
  - ca-certificates=2024.12.31=h06a4308_0
  - ld_impl_linux-64=2.40=h12ee557_0
  - libffi=3.4.4=h6a678d5_1
  - libgcc-ng=11.2.0=h1234567_1
  - libgomp=11.2.0=h1234567_1
  - libstdcxx-ng=11.2.0=h1234567_1
  - ncurses=6.4=h6a678d5_0
  - openssl=3.0.15=h5eee18b_0
  - pip=24.2=py38h06a4308_0
  - python=3.8.20=he870216_0
  - readline=8.2=h5eee18b_0
  - setuptools=75.1.0=py38h06a4308_0
  - sqlite=3.45.3=h5eee18b_0
  - tk=8.6.14=h39e8969_0
  - wheel=0.44.0=py38h06a4308_0
  - xz=5.4.6=h5eee18b_1
  - zlib=1.2.13=h5eee18b_1
  - pip:
      - accelerate==1.0.1
      - certifi==2024.12.14
      - charset-normalizer==3.4.1
      - diffusers==0.32.1
      - filelock==3.16.1
      - fsspec==2024.12.0
      - huggingface-hub==0.27.1
      - idna==3.10
      - importlib-metadata==8.5.0
      - jinja2==3.1.5
      - mpmath==1.3.0
      - networkx==3.1
      - numpy==1.24.4
      - nvidia-cublas-cu12==12.1.3.1
      - nvidia-cuda-cupti-cu12==12.1.105
      - nvidia-cuda-nvrtc-cu12==12.1.105
      - nvidia-cuda-runtime-cu12==12.1.105
      - nvidia-cudnn-cu12==9.1.0.70
      - nvidia-cufft-cu12==11.0.2.54
      - nvidia-curand-cu12==10.3.2.106
      - nvidia-cusolver-cu12==11.4.5.107
      - nvidia-cusparse-cu12==12.1.0.106
      - nvidia-nccl-cu12==2.20.5
      - nvidia-nvjitlink-cu12==12.6.85
      - nvidia-nvtx-cu12==12.1.105
      - pillow==10.4.0
      - protobuf==5.29.3
      - psutil==6.1.1
      - pyyaml==6.0.2
      - regex==2024.11.6
      - requests==2.32.3
      - safetensors==0.5.2
      - sentencepiece==0.2.0
      - sympy==1.13.3
      - tokenizers==0.20.3
      - torch==2.4.1
      - tqdm==4.67.1
      - transformers==4.46.3
      - triton==3.0.0
      - typing-extensions==4.12.2
      - urllib3==2.2.3
      - zipp==3.20.2

Let us know if you encounter other problems, thanks!

[HHHHStar]

Thanks for your reply！ We find the problem is that you used the schnell version while I used the dev version. It is worth mentioning that the schnell accelerated version can even achieve better results.