generate.py

#!/usr/bin/env python3
"""
Generate chunked JSON files from ANSI art collection
Creates an index.json and multiple chunk files for lazy loading
"""

import os
import base64
import json
import math

CHUNK_SIZE = 100  # Files per chunk
OUTPUT_DIR = "public/data"

def main():
    # Ensure output directory exists
    os.makedirs(OUTPUT_DIR, exist_ok=True)

    # Collect all ANSI files
    files = []
    for root, dirs, names in os.walk("ansi", topdown=False):
        for name in names:
            if name[0] != '.':  # Skip hidden files
                path = os.path.join(root, name)
                with open(path, "rb") as f:
                    # Read and remove watermark
                    data = f.read().replace(
                        b"Downloaded From P-80 International Information Systems 304-744-2253",
                        b""
                    )
                    encoded = base64.b64encode(data).decode('ascii')
                    files.append({
                        'name': name,
                        'data': encoded,
                        'size': len(data)
                    })

    # Sort files by name for consistency
    files.sort(key=lambda x: x['name'])

    # Create index
    num_chunks = math.ceil(len(files) / CHUNK_SIZE)
    index = {
        'files': [
            {
                'name': f['name'],
                'chunk': i // CHUNK_SIZE,
                'size': f['size']
            }
            for i, f in enumerate(files)
        ],
        'chunks': num_chunks,
        'total_files': len(files)
    }

    # Write index
    with open(os.path.join(OUTPUT_DIR, 'index.json'), 'w') as f:
        json.dump(index, f, indent=2)

    print(f"Created index.json with {len(files)} files")

    # Write chunks
    for chunk_num in range(num_chunks):
        start = chunk_num * CHUNK_SIZE
        end = min(start + CHUNK_SIZE, len(files))

        chunk_data = {
            f['name']: f['data']
            for f in files[start:end]
        }

        chunk_path = os.path.join(OUTPUT_DIR, f'chunk-{chunk_num}.json')
        with open(chunk_path, 'w') as f:
            json.dump(chunk_data, f)

        print(f"Created chunk-{chunk_num}.json with {len(chunk_data)} files")

    print(f"\nTotal: {num_chunks} chunks, {len(files)} ANSI files")
    print(f"Output directory: {OUTPUT_DIR}")

if __name__ == "__main__":
    main()