diff --git a/samples/hpatches_benchmark.py b/samples/hpatches_benchmark.py new file mode 100644 index 0000000..6e8015f --- /dev/null +++ b/samples/hpatches_benchmark.py @@ -0,0 +1,154 @@ +import argparse +import sys +import logging +from pathlib import Path + +sys.path.append(str(Path(__file__).parent.parent)) # noqa: E402 + +from src.detectors import Detector # noqa: E402 +from src.descriptors import Descriptor # noqa: E402 +from src.matchers import Matcher, OpenCVMatcher # noqa: E402 +from src.feature_matcher import FeatureMatcherCV2 # noqa: E402 + +from samples.utils import build_hpatches_benchmark_config # noqa: E402 +from samples.hpatches_task import HPatchesTask # noqa: E402 +from samples.hpatches_data_manager import HPatchesDataManager # noqa: E402 + +logging.basicConfig(level=logging.INFO, format='[ %(levelname)s ] %(message)s') +logger = logging.getLogger("hpatches_benchmark") + + +def parser(): + arg_parser = argparse.ArgumentParser( + description="HPatches Benchmark Pipeline", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + + available_detectors = list(Detector._METHODS.keys()) + available_descriptors = list(Descriptor._METHODS.keys()) + available_matchers = list(Matcher._METHODS.keys()) + available_matchers_modes = list(OpenCVMatcher._MODE) + available_tasks = list(HPatchesTask._TASKS.keys()) + available_devices = ['cpu', 'cuda', 'mps'] + + arg_parser.add_argument('-det', '--detector', type=str, default='sift', + choices=available_detectors, help='Detector algorithm') + arg_parser.add_argument('-des', '--descriptor', type=str, default='sift', + choices=available_descriptors, help='Descriptor algorithm') + arg_parser.add_argument('-mat', '--matcher', type=str, default='bf', + choices=available_matchers, help='Matching algorithm') + + arg_parser.add_argument('-t', '--tasks', type=str, nargs='+', default=available_tasks, + choices=available_tasks, help='Tasks to evaluate (default: run all available tasks)') + arg_parser.add_argument('-d', '--device', type=str, default=None, + choices=available_devices, help='The device on which the script will be run') + + ds_group = arg_parser.add_argument_group('Dataset config') + ds_group.add_argument('-p', '--path', type=Path, required=True, + help='Path to hpatches-release folder') + ds_group.add_argument('-n', '--num-scenes', type=int, default=116, + help='Number of scenes to process (default: all)') + ds_group.add_argument('-sbs', '--scenes-batch-size', type=int, default=4, + help='Batch size for processing images/scenes') + + task_group = arg_parser.add_argument_group('Task config') + task_group.add_argument('-et', '--eval-thresholds', type=float, nargs='+', default=[5.0], + help='Pixel thresholds for verification (1.0 3.0 5.0 10.0)') + task_group.add_argument('-hm', '--homography-method', type=str, default='ransac', + choices=['ransac', 'magsac', 'lmeds', 'rho'], help='Homography estimation method') + task_group.add_argument('-ht', '--homography-threshold', type=float, default=3.0, + help='Threshold for homography estimation (inlier classification)') + + det_group = arg_parser.add_argument_group('Detector config') + det_group.add_argument('-dn', '--det-nfeatures', type=int, default=None, + help='Max number of features to detect') + det_group.add_argument('-do', '--det-noctave', type=int, default=None, + help='Number of octave layers') + det_group.add_argument('-dt', '--det-threshold', type=float, default=None, + help='Detection threshold') + + des_group = arg_parser.add_argument_group('Descriptor config') + des_group.add_argument('-dsen', '--des-nfeatures', type=int, default=None, + help='Max number of features for descriptor') + des_group.add_argument('-dsdt', '--des-threshold', type=float, default=None, + help='Descriptor threshold') + des_group.add_argument('-dss', '--des-scale', type=float, default=None, + help='Scale factor') + + mat_group = arg_parser.add_argument_group('Matcher config') + mat_group.add_argument('-mat_m', '--matcher_mode', type=str, default='simple', + choices=available_matchers_modes, help='Matching mode') + mat_group.add_argument('-mr', '--mat-ratio', type=float, default=None, + help='Ratio threshold for KNN') + mat_group.add_argument('-mc', '--mat-cross-check', action='store_true', default=None, + help='Enable cross-check for BF matcher') + return arg_parser.parse_args() + + +def main(): + args = parser() + + try: + if not args.path.exists(): + logger.error(f"Dataset path does not exist: {args.path}") + return 1 + + logger.info(f"Starting HPatches Benchmark. Tasks: {args.tasks}, Thresholds: {args.eval_thresholds}") + + config = build_hpatches_benchmark_config(args) + feature_matcher = FeatureMatcherCV2(detector=args.detector, descriptor=args.descriptor, + matcher=args.matcher, logger=logger, config=config) + dm = HPatchesDataManager(logger=logger, config=config['dataset']) + + task_objects = {} + results_by_task = {} + for task_name in args.tasks: + task_objects[task_name] = HPatchesTask.create(task_name=task_name, logger=logger, + config=config['task']) + results_by_task[task_name] = {} + + while dm.has_more_data(): + current_batch = dm.load_batch() + if not current_batch: + break + + for scene_name, data in current_batch.items(): + scene_matching_data = {scene_name: {}} + img_ref = data['ref_img'] + + for i, target in data['targets'].items(): + img_tgt = target['image'] + H = target['H'] + + features_ref, features_tgt, correspondences = feature_matcher.match(img_ref, img_tgt) + scene_matching_data[scene_name][i] = { + 'kp_ref': features_ref['kp'], + 'kp_tgt': features_tgt['kp'], + 'matches': correspondences['matches'], + 'H': H, + 'ref_shape': data['ref_shape'], + 'tgt_shape': target['tgt_shape'], + } + + for task_name, task_obj in task_objects.items(): + results_scene = task_obj.eval_task(scene_matching_data, [scene_name]) + if task_name not in results_by_task: + results_by_task[task_name] = results_scene + else: + for threshold in results_scene: + if threshold not in results_by_task[task_name]: + results_by_task[task_name][threshold] = {} + results_by_task[task_name][threshold].update(results_scene[threshold]) + + for task_name, task_obj in task_objects.items(): + task_obj.report_metrics(results_by_task[task_name], f"End-to-End Pipeline [{task_name}]") + + logger.info("Pipeline finished successfully") + return 0 + + except Exception as e: + logger.exception(f"An error occurred: {e}") + return 1 + + +if __name__ == '__main__': + sys.exit(main() or 0) diff --git a/samples/hpatches_data_manager.py b/samples/hpatches_data_manager.py new file mode 100644 index 0000000..dfdafe9 --- /dev/null +++ b/samples/hpatches_data_manager.py @@ -0,0 +1,88 @@ +import numpy as np +from pathlib import Path + +from src.image_utils import read_image + + +class HPatchesDataManager: + _ref_filename = "1.ppm" + _img_indices = [2, 3, 4, 5, 6] + + def __init__(self, logger, config=None): + if config is None: + config = {} + + self._raw_data_path = Path(config.pop("raw_data_path", "hpatches-sequences-release")) + self._num_scenes = config.pop("num_scenes", 116) + self._scenes_batch_size = config.pop("scenes_batch_size", 4) + self._logger = logger + + self._current_idx = 0 + self.all_scenes = self._get_all_scenes() + + def _get_all_scenes(self): + scenes = [d for d in self._raw_data_path.iterdir() if d.is_dir()] + if self._num_scenes: + scenes = scenes[:self._num_scenes] + return scenes + + def has_more_data(self): + return self._current_idx < len(self.all_scenes) + + def _load_single_scene(self, scene_dir): + ref_path = scene_dir / self._ref_filename + img_ref = read_image(ref_path) + + if img_ref is None: + self._logger.warning(f"Could not read reference image in {scene_dir}") + return None + + scene_data = { + 'ref_img': img_ref, + 'ref_shape': img_ref.shape, + 'targets': {}, + 'name': scene_dir.name + } + + for i in self._img_indices: + target_path = scene_dir / f"{i}.ppm" + img_target = read_image(target_path) + h_path = scene_dir / f"H_1_{i}" + + if img_target is not None and h_path.exists(): + H = np.loadtxt(str(h_path)) + scene_data['targets'][i] = { + 'image': img_target, + 'H': H, + 'tgt_shape': img_target.shape + } + return scene_data + + def load_batch(self): + if not self.has_more_data(): + self._logger.info("No more scenes to process.") + return None + + if not self._current_idx: + self._logger.info(f"Loading {self._num_scenes} scenes") + + start = self._current_idx + end = min(start + self._scenes_batch_size, len(self.all_scenes)) + + batch_paths = self.all_scenes[start: end] + batch_data = {} + + for scene_dir in batch_paths: + self._logger.info(f"Loading {scene_dir}") + scene_content = self._load_single_scene(scene_dir) + if scene_content: + batch_data[scene_dir.name] = scene_content + + self._current_idx = end + if self._current_idx == len(self.all_scenes): + self._logger.info(f"Loaded {self._num_scenes} scenes") + + return batch_data + + def reset(self): + self._current_idx = 0 diff --git a/samples/hpatches_table_benchmark.py b/samples/hpatches_table_benchmark.py new file mode 100644 index 0000000..cfdb644 --- /dev/null +++ b/samples/hpatches_table_benchmark.py @@ -0,0 +1,279 @@ +import argparse +import re +import subprocess +import sys +import logging +import pandas as pd +from pathlib import Path + +sys.path.append(str(Path(__file__).parent.parent)) # noqa: E402 + +from src.algorithms import DETECTOR_DESCRIPTOR_COMPATIBILITY, DESCRIPTOR_MATCHER_COMPATIBILITY # noqa: E402 + +logging.basicConfig(level=logging.INFO, format='[ %(levelname)s ] %(message)s') +logger = logging.getLogger("HPatchesTableBenchmark") + + +def parse_metrics_log(log_output, task_names, thresholds): + results = {} + + for threshold in thresholds: + patterns = { + 'matchingap': { + f'matchingap_mean_ap_{threshold}': ( + rf'--- END-TO-END PIPELINE \[MATCHINGAP\] @ {threshold}(?:\.0)?px ---' + rf'[\s\S]*?Mean Total AP:\s*([\d\.]+)' + ), + }, + 'matchingscore': { + f'matchingscore_mean_ms_{threshold}': ( + rf'--- END-TO-END PIPELINE \[MATCHINGSCORE\] @ {threshold}(?:\.0)?px ---' + rf'[\s\S]*?Mean MS:\s*([\d\.]+)' + ), + f'matchingscore_mean_prec_{threshold}': ( + rf'--- END-TO-END PIPELINE \[MATCHINGSCORE\] @ {threshold}(?:\.0)?px ---' + rf'[\s\S]*?Mean Prec:\s*([\d\.]+)' + ), + }, + 'homographyauc': { + f'homographyauc_mean_auc_{threshold}': ( + rf'--- END-TO-END PIPELINE \[HOMOGRAPHYAUC\] @ {threshold}(?:\.0)?px ---' + rf'[\s\S]*?Mean AUC:\s*([\d\.]+)' + ), + } + } + + for task_name in task_names: + task_patterns = patterns.get(task_name.lower(), {}) + for key, pattern in task_patterns.items(): + match = re.search(pattern, log_output, re.IGNORECASE) + if match: + results[key] = float(match.group(1)) + else: + logger.debug(f"No match for: {key}") + + return results + + +def run_benchmark(detector, descriptor, matcher, dataset_path, tasks, thresholds, device='cpu', num_scenes=None): + cmd = [ + sys.executable, '-m', 'samples.hpatches_benchmark', + '-det', detector, + '-des', descriptor, + '-mat', matcher, + '-t'] + tasks + [ + '-p', str(dataset_path), + '-et'] + [str(t) for t in thresholds] + + if device: + cmd.extend(['-d', device]) + + if num_scenes: + cmd.extend(['-n', str(num_scenes)]) + + logger.info(f"Running: {detector}+{descriptor}+{matcher}") + + process = None + try: + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True + ) + stdout, stderr = process.communicate() + output = stdout + stderr + + if process.returncode != 0: + logger.warning(f"Failed: {detector}+{descriptor}+{matcher}") + logger.debug(f"Error output: {stderr}") + return False, {} + + all_results = parse_metrics_log(output, tasks, thresholds) + return True, all_results + + except Exception as e: + logger.warning(f"Error: {e}") + return False, {} + finally: + if process is not None and process.poll() is None: + process.kill() + process.wait() + + +def get_all_combinations(): + combinations = [] + + for detector in DETECTOR_DESCRIPTOR_COMPATIBILITY: + descriptors = DETECTOR_DESCRIPTOR_COMPATIBILITY.get(detector, []) + + for descriptor in descriptors: + matchers = DESCRIPTOR_MATCHER_COMPATIBILITY.get(descriptor, []) + + for matcher in matchers: + combinations.append((detector, descriptor, matcher)) + + return combinations + + +def load_existing_results(output_path): + if not output_path.exists(): + logger.info(f"No existing results found at {output_path}") + return None, set() + + try: + df = pd.read_csv(output_path) + logger.info(f"Loaded {len(df)} existing results from {output_path}") + + existing_combos = set() + for _, row in df.iterrows(): + combo = (row['detector'], row['descriptor'], row['matcher']) + existing_combos.add(combo) + + logger.info(f"Found {len(existing_combos)} unique combinations already computed") + return df, existing_combos + + except Exception as e: + logger.warning(f"Error loading existing results: {e}") + return None, set() + + +def save_single_result(output_path, new_result, tasks, thresholds): + base_columns = ['detector', 'descriptor', 'matcher', 'device', 'num_scenes'] + + metric_columns = [] + for task in tasks: + if task.lower() == 'matchingap': + for threshold in thresholds: + metric_columns.append(f'matchingap_mean_ap_{threshold}') + elif task.lower() == 'matchingscore': + for threshold in thresholds: + metric_columns.append(f'matchingscore_mean_ms_{threshold}') + metric_columns.append(f'matchingscore_mean_prec_{threshold}') + elif task.lower() == 'homographyauc': + for threshold in thresholds: + metric_columns.append(f'homographyauc_mean_auc_{threshold}') + + columns_order = base_columns + metric_columns + + if output_path.exists(): + df = pd.read_csv(output_path) + else: + df = pd.DataFrame(columns=columns_order) + + new_df = pd.DataFrame([new_result]) + df = pd.concat([df, new_df], ignore_index=True) + + existing_columns = [col for col in columns_order if col in df.columns] + df = df[existing_columns] + + output_path.parent.mkdir(parents=True, exist_ok=True) + df.to_csv(output_path, index=False) + + +def table_benchmark(dataset_path, output_csv, tasks, thresholds, device='cpu', num_scenes=None, skip_existing=True): + combinations = get_all_combinations() + logger.info(f"Found {len(combinations)} valid combinations") + logger.info(f"Tasks to run: {tasks}") + logger.info(f"Thresholds: {thresholds}") + + existing_df, existing_combos = load_existing_results(output_csv) + + if skip_existing and existing_combos: + remaining_combos = [c for c in combinations if c not in existing_combos] + logger.info(f"Skipping {len(existing_combos)} already computed combinations") + logger.info(f"Remaining: {len(remaining_combos)} combinations") + combinations = remaining_combos + + if not combinations: + logger.info("All combinations already computed!") + return + + for detector, descriptor, matcher in combinations: + logger.info(f"Processing: {detector}+{descriptor}+{matcher}") + + combo_result = { + 'detector': detector, + 'descriptor': descriptor, + 'matcher': matcher, + 'device': device, + 'num_scenes': num_scenes if num_scenes else 116, + } + + success, metrics = run_benchmark( + detector, descriptor, matcher, + dataset_path, tasks, thresholds, device, num_scenes + ) + + if not success: + logger.warning(f"Failed: {detector}+{descriptor}+{matcher}") + continue + + combo_result.update(metrics) + + try: + save_single_result(output_csv, combo_result, tasks, thresholds) + logger.info(f"Saved: {detector}+{descriptor}+{matcher}") + logger.info(f"Metrics: {metrics}") + except Exception as e: + logger.error(f"Error saving result: {e}") + continue + + logger.info("All combinations processed") + logger.info(f"Results saved to: {output_csv}") + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Run HPatches benchmarks for all combinations with incremental saving", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + + available_devices = ['cpu', 'cuda', 'mps'] + available_tasks = ['matchingap', 'matchingscore', 'homographyauc'] + + parser.add_argument('-p', '--path', type=Path, required=True, + help='Path to hpatches-sequences-release folder') + parser.add_argument('-o', '--output', type=Path, default=Path('hpatches_results.csv'), + help='Output CSV file path') + + parser.add_argument('-t', '--tasks', type=str, nargs='+', choices=available_tasks, + default=['matchingap', 'matchingscore', 'homographyauc'], + help='Tasks to run') + parser.add_argument('-et', '--eval-thresholds', type=float, nargs='+', default=[5.0], + help='Pixel thresholds (1.0 3.0 5.0 10.0)') + + parser.add_argument('-d', '--device', type=str, default=None, choices=available_devices, + help='Device to run on') + parser.add_argument('-n', '--num-scenes', type=int, default=116, + help='Number of scenes to process') + parser.add_argument('--no-skip', action='store_true', + help='Recompute already existing combinations') + + return parser.parse_args() + + +def main(): + args = parse_args() + + if not args.path.exists(): + logger.error(f"Dataset path does not exist: {args.path}") + return 1 + + logger.info("HPatches Benchmark Table Generator") + + table_benchmark( + dataset_path=args.path, + output_csv=args.output, + tasks=args.tasks, + thresholds=args.eval_thresholds, + device=args.device, + num_scenes=args.num_scenes, + skip_existing=not args.no_skip, + ) + + logger.info("Benchmark completed successfully") + return 0 + + +if __name__ == '__main__': + sys.exit(main() or 0) diff --git a/samples/hpatches_task.py b/samples/hpatches_task.py new file mode 100644 index 0000000..c57541d --- /dev/null +++ b/samples/hpatches_task.py @@ -0,0 +1,314 @@ +import cv2 as cv +import numpy as np +from abc import ABC, abstractmethod + + +class HPatchesTask(ABC): + _TASKS = {} + _img_indices = [2, 3, 4, 5, 6] + + def __init__(self, logger): + self._logger = logger + + def __init_subclass__(cls, register=True, **kwargs): + super().__init_subclass__(**kwargs) + + if register: + key = cls.__name__.replace("Task", "").lower() + if key: + HPatchesTask._TASKS[key] = cls + + @classmethod + def create(cls, task_name, logger, config=None): + if config is None: + config = {} + + task_class = cls._TASKS.get(task_name.lower()) + if not task_class: + raise ValueError(f"Unknown task: {task_name}. Available: {list(cls._TASKS.keys())}") + + return task_class(logger, config) + + @abstractmethod + def eval_task(self, descriptors, split): + pass + + @classmethod + def _tpfp(cls, scores, labels, numpos=None): + p = int(np.sum(labels)) + n = len(labels) - p + + if numpos is not None: + assert (numpos >= p), 'numpos smaller that number of positives in labels' + extra_pos = numpos - p + p = numpos + + scores = np.hstack((scores, np.repeat(-np.inf, extra_pos))) + labels = np.hstack((labels, np.repeat(1, extra_pos))) + + perm = np.argsort(-scores, kind='mergesort', axis=0) + scores = scores[perm] + stop = np.max(np.where(scores > -np.inf)) + perm = perm[0:stop + 1] + labels = labels[perm] + + tp = np.hstack((0, np.cumsum(labels == 1))) + fp = np.hstack((0, np.cumsum(labels == 0))) + return tp, fp, p, n, perm + + @classmethod + def _pr(cls, scores, labels, numpos=None): + [tp, fp, p, n, perm] = cls._tpfp(scores, labels, numpos) + + small = 1e-10 + recall = tp / float(np.maximum(p, small)) + precision = np.maximum(tp, small) / np.maximum(tp + fp, small) + return precision, recall, np.trapezoid(precision, recall) + + +class BaseMatchingTask(HPatchesTask, register=False): + def __init__(self, logger, config): + super().__init__(logger) + + self._eval_thresholds = config.get('eval_thresholds', [5.0]) + if not isinstance(self._eval_thresholds, list): + self._eval_thresholds = [self._eval_thresholds] + + def _compute_numpos(self, data, threshold): + kp_ref = data['kp_ref'] + kp_tgt = data['kp_tgt'] + H = data['H'] + tgt_shape = data['tgt_shape'] + + if len(kp_ref) == 0 or len(kp_tgt) == 0: + return 0 + + pts_ref = np.array([kp.pt for kp in kp_ref], dtype=np.float32).reshape(-1, 1, 2) + pts_tgt_gt = cv.perspectiveTransform(pts_ref, H).reshape(-1, 2) + + if tgt_shape is not None: + h, w = tgt_shape[:2] + valid_mask = ((pts_tgt_gt[:, 0] >= 0) & (pts_tgt_gt[:, 0] < w) + & (pts_tgt_gt[:, 1] >= 0) & (pts_tgt_gt[:, 1] < h)) + pts_tgt_gt = pts_tgt_gt[valid_mask] + + if len(pts_tgt_gt) == 0: + return 0 + + pts_tgt = np.array([kp.pt for kp in kp_tgt], dtype=np.float32) + + num_gt = 0 + for pt_gt in pts_tgt_gt: + pixel_dists = np.linalg.norm(pts_tgt - pt_gt, axis=1) + if len(pixel_dists) > 0 and np.min(pixel_dists) <= threshold: + num_gt += 1 + + return num_gt + + def _get_match_results(self, data, threshold): + kp_ref = data['kp_ref'] + kp_tgt = data['kp_tgt'] + matches = data['matches'] + H = data['H'] + + if not matches: + return None + + pts_ref = np.array([kp_ref[m.queryIdx].pt for m in matches], dtype=np.float32).reshape(-1, 1, 2) + pts_tgt_pred = np.array([kp_tgt[m.trainIdx].pt for m in matches], dtype=np.float32).reshape(-1, 1, 2) + pts_tgt_gt = cv.perspectiveTransform(pts_ref, H) + + pixel_dists = np.linalg.norm(pts_tgt_pred - pts_tgt_gt, axis=2).flatten() + labels = (pixel_dists <= threshold) + descriptor_dists = np.array([m.distance for m in matches], dtype=np.float32) + + if descriptor_dists.size > 0: + max_dist = descriptor_dists.max() + scores = 1.0 - (descriptor_dists / max_dist) + else: + scores = np.array([], dtype=np.float32) + + return { + 'labels': labels, + 'distances': pixel_dists, + 'num_kp_ref': len(kp_ref), + 'num_kp_tgt': len(kp_tgt), + 'num_matches': len(matches), + 'scores': scores + } + + +class MatchingAPTask(BaseMatchingTask): + def eval_task(self, matching_data, split): + results = {threshold: {seq: {} for seq in split} for threshold in self._eval_thresholds} + + for threshold in self._eval_thresholds: + self._logger.info(f'Evaluating Feature Matching (mAP) @ {threshold}px') + + for seq in split: + if seq not in matching_data: + continue + + for i in self._img_indices: + data = matching_data[seq].get(i) + res = self._get_match_results(data, threshold) if data else None + + if res is not None: + _, _, ap = self._pr(res['scores'], res['labels'], + numpos=self._compute_numpos(data, threshold)) + results[threshold][seq][i] = {'ap': ap} + + return results + + def report_metrics(self, results, task_name="Feature Matching (mAP)"): + for threshold, threshold_results in results.items(): + all_ap_values = [ + img_data['ap'] + for scene_data in threshold_results.values() + for img_data in scene_data.values() + if 'ap' in img_data + ] + + if not all_ap_values: + self._logger.warning(f"No AP results found for threshold {threshold}px") + continue + + mean_total_ap = np.mean(all_ap_values) + self._logger.info(f"--- {task_name.upper()} @ {threshold}px ---") + self._logger.info(f"Mean Total AP: {mean_total_ap:.4f}") + + +class MatchingScoreTask(BaseMatchingTask): + def eval_task(self, matching_data, split): + results = {threshold: {seq: {} for seq in split} for threshold in self._eval_thresholds} + + for threshold in self._eval_thresholds: + self._logger.info(f'Evaluating Matching Score & Precision @ {threshold}px') + + for seq in split: + if seq not in matching_data: + continue + + for i in self._img_indices: + data = matching_data[seq].get(i) + res = self._get_match_results(data, threshold) if data else None + + if res is not None: + num_inliers = np.sum(res['labels']) + results[threshold][seq][i] = { + 'ms': num_inliers / min(res['num_kp_ref'], res['num_kp_tgt']), + 'prec': num_inliers / res['num_matches'] if res['num_matches'] > 0 else 0 + } + + return results + + def report_metrics(self, results, task_name="Matching Score & Precision"): + for threshold, threshold_results in results.items(): + all_ms_values = [ + img_data['ms'] + for scene_data in threshold_results.values() + for img_data in scene_data.values() + if 'ms' in img_data + ] + + if not all_ms_values: + self._logger.warning(f"No MS results for threshold {threshold}px") + continue + + all_prec_values = [ + img_data['prec'] + for scene_data in threshold_results.values() + for img_data in scene_data.values() + if 'prec' in img_data + ] + + mean_total_ms = np.mean(all_ms_values) + mean_total_prec = np.mean(all_prec_values) + + self._logger.info(f"--- {task_name.upper()} @ {threshold}px ---") + self._logger.info(f"Mean MS: {mean_total_ms:.4f}, Mean Prec: {mean_total_prec:.4f}") + + +class HomographyAUCTask(HPatchesTask): + _HOMOGRAPHY_METHODS = { + "ransac": cv.RANSAC, + "magsac": cv.USAC_MAGSAC, + "lmeds": cv.LMEDS, + "rho": cv.RHO + } + + def __init__(self, logger, config): + super().__init__(logger) + + self._eval_thresholds = config.pop('eval_thresholds', [5.0]) + if not isinstance(self._eval_thresholds, list): + self._eval_thresholds = [self._eval_thresholds] + + self._homography_threshold = config.pop('homography_threshold', 3.0) + self._homography_method = config.pop('homography_method', "ransac") + + def eval_task(self, matching_data, split): + results = {threshold: {seq: {} for seq in split} for threshold in self._eval_thresholds} + + for threshold in self._eval_thresholds: + self._logger.info(f'Evaluating Homography AUC @ {threshold}px') + + for seq in split: + if seq not in matching_data: + continue + + for i in self._img_indices: + data = matching_data[seq].get(i) + if not data or not data['matches']: + continue + + kp_ref = data['kp_ref'] + kp_tgt = data['kp_tgt'] + matches = data['matches'] + + pts_ref = np.array([kp_ref[m.queryIdx].pt for m in matches], + dtype=np.float32).reshape(-1, 1, 2) + pts_tgt_pred = np.array([kp_tgt[m.trainIdx].pt for m in matches], + dtype=np.float32).reshape(-1, 1, 2) + + if len(pts_ref) < 4: + continue + + H_gt = data['H'] + H_pred, mask = cv.findHomography(pts_ref, pts_tgt_pred, + self._HOMOGRAPHY_METHODS[self._homography_method], + self._homography_threshold) + + if H_pred is None: + continue + + h, w = data['ref_shape'][:2] + corners = np.float32([[0, 0], [w, 0], [w, h], [0, h]]).reshape(-1, 1, 2) + + corners_gt = cv.perspectiveTransform(corners, H_gt) + corners_pred = cv.perspectiveTransform(corners, H_pred) + + error = np.mean(np.linalg.norm(corners_gt - corners_pred, axis=2)) + results[threshold][seq][i] = {'error': error} + + return results + + def report_metrics(self, results, task_name="Homography AUC"): + for threshold, threshold_results in results.items(): + all_errors = [ + img['error'] + for s in threshold_results.values() + for img in s.values() + if 'error' in img + ] + + if not all_errors: + self._logger.warning(f"No results for threshold {threshold}px") + continue + + thresholds = np.linspace(0, threshold, 100) + acc_curve = [np.mean(np.array(all_errors) < t) for t in thresholds] + + global_auc = np.trapezoid(acc_curve, thresholds) / threshold + self._logger.info(f"--- {task_name.upper()} @ {threshold}px ---") + self._logger.info(f"Mean AUC: {global_auc:.4f}") diff --git a/samples/utils.py b/samples/utils.py index dcbd93c..abf9d2d 100644 --- a/samples/utils.py +++ b/samples/utils.py @@ -129,17 +129,19 @@ def build_hpatches_dataset_config(args): config['path'] = args.path if args.num_scenes is not None: config['num_scenes'] = args.num_scenes - if args.batch_size is not None: - config['batch_size'] = args.batch_size + if args.scenes_batch_size is not None: + config['scenes_batch_size'] = args.scenes_batch_size return config def build_hpatches_task_config(args): config = dict() - if args.pixel_threshold is not None: - config['pixel_threshold'] = args.pixel_threshold + if args.eval_thresholds is not None: + config['eval_thresholds'] = args.eval_thresholds if args.homography_method is not None: config['homography_method'] = args.homography_method + if args.homography_threshold is not None: + config['homography_threshold'] = args.homography_threshold return config diff --git a/src/super_point.py b/src/super_point.py index d398ae5..36090e8 100644 --- a/src/super_point.py +++ b/src/super_point.py @@ -121,4 +121,4 @@ def compute(self, img, features): return self._forward(img) def detectAndCompute(self, img): - return self._forward(img) + return self._forward(img) \ No newline at end of file