Metadata recorded frame reproduction of bug (most of the time)

[alifuaziz]

Sometimes the number of recorded frames in the metadata is different to the number of frames that are actually recorded.

The following the config file

{
    "application_config": {
        "gui_config": {
            "camera_update_rate": 20,
            "camera_updates_per_display_update": 4,
            "font_size": 12
        },
        "ffmpeg_config": {
            "crf": 23,
            "encoding_speed": "slow",
            "compression_standard": "h265"
        },
        "paths_config": {
            "ROOT": "C:\\Users\\alifa\\Documents\\pyMV-Local\\code",
            "camera_dir": "C:\\Users\\alifa\\Documents\\pyMV-Local\\code\\config",
            "encoder_dir": "C:\\Users\\alifa\\Documents\\pyMV-Local\\code\\config",
            "data_dir": "C:\\Users\\alifa\\Documents\\pyMV-Local\\code\\data",
            "config_dir": "C:\\Users\\alifa\\Documents\\pyMV-Local\\code\\config",
            "icons_dir": "C:\\Users\\alifa\\Documents\\pyMV-Local\\code\\GUI\\icons"
        },
        "default_camera_config": {
            "name": null,
            "fps": 60,
            "downsampling_factor": 1,
            "exposure_time": 15000,
            "gain": 0,
            "external_trigger": false,
            "pixel_format": "Mono"
        }
    },
    "experiment_config": {
        "data_dir": "data\\test-photo-1\\config_ncams_7_downsample_1_fps_60_update_20_upd_per_disp_4_crf_23_speed_slow_comp_h265",
        "n_cameras": 7,
        "n_columns": 1,
        "cameras": [
            {
                "label": "18360349-spinnaker",
                "subject_id": "recording-1"
            },
            {
                "label": "16369381-spinnaker",
                "subject_id": "recording-2"
            },
            {
                "label": "16369379-spinnaker",
                "subject_id": "recording-3"
            },
            {
                "label": "21013409-spinnaker",
                "subject_id": "recording-4"
            },
            {
                "label": "18360348-spinnaker",
                "subject_id": "recording-5"
            },
            {
                "label": "21013402-spinnaker",
                "subject_id": "recording-6"
            },
            {
                "label": "21013401-spinnaker",
                "subject_id": "recording-7"
            }
        ]
    },
    "camera_config": [
        {
            "name": null,
            "unique_id": "18360349-spinnaker",
            "fps": 60,
            "exposure_time": 15666,
            "gain": 0,
            "pixel_format": "Mono",
            "downsampling_factor": 1
        },
        {
            "name": null,
            "unique_id": "16369381-spinnaker",
            "fps": 60,
            "exposure_time": 15666,
            "gain": 0,
            "pixel_format": "Mono",
            "downsampling_factor": 1
        },
        {
            "name": null,
            "unique_id": "16369379-spinnaker",
            "fps": 60,
            "exposure_time": 15666,
            "gain": 0,
            "pixel_format": "Mono",
            "downsampling_factor": 1
        },
        {
            "name": null,
            "unique_id": "21013409-spinnaker",
            "fps": 60,
            "exposure_time": 15666,
            "gain": 0,
            "pixel_format": "Mono",
            "downsampling_factor": 1
        },
        {
            "name": null,
            "unique_id": "18360348-spinnaker",
            "fps": 60,
            "exposure_time": 15666,
            "gain": 0,
            "pixel_format": "Mono",
            "downsampling_factor": 1
        },
        {
            "name": null,
            "unique_id": "21013402-spinnaker",
            "fps": 60,
            "exposure_time": 15666,
            "gain": 0,
            "pixel_format": "Mono",
            "downsampling_factor": 1
        },
        {
            "name": null,
            "unique_id": "21013401-spinnaker",
            "fps": 60,
            "exposure_time": 15666,
            "gain": 0,
            "pixel_format": "Mono",
            "downsampling_factor": 1
        }
    ],
    "record-on-startup": true,
    "close_after": "00:30"
}

Used with the frame_counting_checking.py can produce the output:

Frame count mismatch. Recorded: 880, Actual: 884       
Frame count mismatch. Recorded: 965, Actual: 969       
Frame count matches recorded frame count.
Frame count matches recorded frame count.
Frame count matches recorded frame count.
Frame count matches recorded frame count.
Frame count matches recorded frame count.

from pathlib import Path
import json, sys, subprocess

from metadata_validation_script import get_video_frame_count

# config_path = r"C:\Users\alifa\Documents\pyMV-Local\code\data\test-photo-1\config_ncams_6_downsample_1_fps_120_update_20_upd_per_disp_2_crf_23_speed_slow_comp_h265\test_config.json"
config_path = r"C:\Users\alifa\Documents\pyMV-Local\code\data\test-photo-1\config_ncams_7_downsample_1_fps_60_update_20_upd_per_disp_4_crf_23_speed_slow_comp_h265\test_config.json"
# Load the JSON config file
with open(config_path, "r") as f:
    config_data = json.load(f)

# Construct the command as a list of arguments
command = [
    sys.executable,  # Python executable
    Path(".") / "pyMultiVideo_GUI.pyw",  # Path to GUI
    # Test options specified
    "--experiment-config",
    json.dumps(json.dumps(config_data["experiment_config"])),  # Config file passed as JSON formatted string
    "--camera-config",
    json.dumps(json.dumps(config_data["camera_config"])),  # Config file passed as JSON formatted string
    "--application-config",
    json.dumps(json.dumps(config_data["application_config"])),  # Config file passed as JSON formatted string
    "--record-on-startup",
    config_data["record-on-startup"],  # Application records on startup
    "--close-after",
    config_data["close_after"],  # Time after which the application closes
]

# Join the list into a single string with spaces between each element
command = " ".join(map(str, command))
print(command)
# Start the process
process = subprocess.Popen(
    command, stdin=subprocess.PIPE, creationflags=subprocess.CREATE_NEW_PROCESS_GROUP
)  # Ensure it runs in a new process group

process.wait()

# Get all mp4 files in the folder containing the config file
mp4_files = list(Path(config_path).parent.glob("*.mp4"))

for mp4 in mp4_files:
    metadata_file = mp4.parent / (mp4.stem + "_metadata.json")
    with open(metadata_file, "r") as f:
        metadata = json.load(f)
    frame_count = get_video_frame_count(mp4)
    if metadata_file.exists():
        recorded_frames = metadata["recorded_frames"]
        if frame_count == recorded_frames:
            print("Frame count matches recorded frame count.")
        else:
            print(f"Frame count mismatch. Recorded: {recorded_frames}, Actual: {frame_count}")

[alifuaziz]

The following script was used to make this test.

import numpy as np
from concurrent.futures import ThreadPoolExecutor
import subprocess
import os, fire, time
from pathlib import Path
from pyinstrument import Profiler
import csv


class SimulatedCamera:
    def __init__(self, height, width, n_images):
        self.height = height
        self.width = width
        self.n_images = n_images
        self.images = [
            np.random.randint(0, 256, (self.height, self.width), dtype=np.uint8) for _ in range(self.n_images)
        ]
        self.index = 0

    def get_next_image(self):
        if self.index >= self.n_images:
            self.index = 0  # Loop back to start
        img = self.images[self.index]
        self.index += 1
        return img


def create_ffmpeg_process(i, width, height, fps, terminal_output=False):
    cmd = [
        "ffmpeg",
        "-y",
        "-f",
        "rawvideo",
        "-vcodec",
        "rawvideo",
        "-pix_fmt",
        "gray",
        "-s",
        f"{width}x{height}",
        "-r",
        str(fps),
        "-I",
        "-",
        "-c:v",
        "hevc_nvenc",  # h265 encoding
        f"test/data/output_camera_{i}.mp4",
    ]
    if terminal_output:
        return subprocess.Popen(cmd, stdin=subprocess.PIPE)
    else:
        return subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)


def get_video_frame_count(video_path):
    """Check the number of frames in a video
    video_path: path to the mp4 file"""
    cmd = [
        "ffprobe",
        "-v",
        "error",
        "-select_streams",
        "v:0",
        "-count_frames",
        "-show_entries",
        "stream=nb_read_frames",
        "-of",
        "default=nokey=1:noprint_wrappers=1",
        video_path,
    ]
    result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
    try:
        return int(result.stdout.strip())
    except ValueError:
        return None


def main(
    n_images_generated=5,
    height=1000,
    width=1000,
    fps=30,
    n_camera=1,
    submit_number=200,
    submit_interval=0.01,
    buffer_size=200,
    check_frame_count=True,
    clear_mp4=True,
    terminal_output=False,
):
    """
    Args:
        n_images_generated (int): the number of images of random noise that are generated
        height (int): Vertical resolution of frame
        width (int): Horizontal resolution of frame
        submit_number (int): The number of times the submit function is called per camera
        submit_interval (int): The time between submts of jobs into the threadpool
        clear_mp4 (bool): Remove the mp4s generated if True
    """
    os.makedirs("test/data/", exist_ok=True)

    profiler = Profiler()
    profiler.start()

    print("Starting Threads...")
    executor = ThreadPoolExecutor(max_workers=32)

    print("Creating Cameras...")
    cameras = [SimulatedCamera(height, width, n_images_generated) for _ in range(n_camera)]

    print("Opening FFMPEG Processes...")
    ffmpeg_processes = [create_ffmpeg_process(i, width, height, fps, terminal_output) for i in range(n_camera)]
    time.sleep(2)
    # List to keep track of number of frames recorded per ffmpeg process
    frames_recorded = [0 for _ in range(n_camera)]

    def stream_camera(cam, proc, n_frames):
        for _ in range(n_frames):
            img = cam.get_next_image()
            proc.stdin.write(img.tobytes())

    # Submit data via executor
    print("Submitting FFMPEG jobs...")
    futures = []
    for i in range(submit_number):
        for cam, proc in zip(cameras, ffmpeg_processes):
            future = executor.submit(stream_camera, cam, proc, buffer_size)
            futures.append(future)
            frames_recorded[cameras.index(cam)] += buffer_size
            print("Futures Length:", len(futures))
            futures = [f for f in futures if not f.done()]
        time.sleep(submit_interval)

        # Remove completed futures
    # Overflowing the number of jobs being submitted to the threadpool
    # Monitering the number of jobs currently in the threadpool
    # Ensureing that the threadpool is empty before closing the application
    print("Monitor Futures Length...")
    while len(futures) != 0:
        futures = [f for f in futures if not f.done()]
        print(len(futures))
        time.sleep(submit_interval)
    # Close FFMPEG process
    for proc in ffmpeg_processes:
        proc.stdin.close()
        proc.wait()

    profiler.stop()
    profiler_output_path = "test/data/profiler_report.html"
    profiler.write_html(profiler_output_path)
    print(f"Profiler HTML report saved to {profiler_output_path}")

    print("Finished Encoding.")
    if check_frame_count:
        print("Checking Frame Count...")
        with open("test/data/frame_count_log.csv", "w", newline="") as csvfile:
            writer = csv.writer(csvfile)
            writer.writerow(["Camera", "Expected Frames", "Actual Frames", "Match"])
            for i in range(n_camera):
                video_path = f"test/data/output_camera_{i}.mp4"
                expected_frames = frames_recorded[I]
                actual_frames = get_video_frame_count(video_path)
                match = actual_frames == expected_frames
                writer.writerow([i, expected_frames, actual_frames, match])
                print(
                    f"Camera {i}: Expected frames = {expected_frames}, Actual frames = {actual_frames}, Match = {match}"
                )
                if not match:
                    print(f"Warning: Frame count mismatch for camera {i}!")
    else:
        with open("test/data/frame_count_log.csv", "w", newline="") as csvfile:
            writer = csv.writer(csvfile)
            writer.writerow(["Camera", "Expected Frames"])
            for i in range(n_camera):
                writer.writerow([i, frames_recorded[I]])
                print(f"Camera {i}: Expected frames = {frames_recorded[i]}")

    if clear_mp4:
        print("Removing MP4 Files from data folder")
        data_dir = Path("test/data/")
        for file in data_dir.glob("*.mp4"):
            file.unlink()

    print("Done")


if __name__ == "__main__":
    fire.Fire(main)

[alifuaziz]

By ensuring that all the thread pool workers have been completed before exiting application, the number of dropped frames recorded is always correct, after testing on one setup.

[ThomasAkam]

@alifuaziz is this issue fixed? was closed then reopened.