From 55e36925c9259edfd794f0c0dad6baa7377c4557 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Artur=20Mendon=C3=A7a=20Arruda?= Date: Thu, 1 Oct 2026 01:46:13 -0300 Subject: [PATCH 01/14] feat(neural): resolve every population by FlyWire cell type and add LC4 to the eye --- src/config.py | 95 ++++++++++++++++++--------------- src/neural/cell_types.py | 59 ++++++++++++++++++++ src/night/engine.py | 69 ++++++++++++------------ src/night/neurons.py | 55 +++++++++++++++++++ src/night/state.py | 6 +++ tests/test_cell_types.py | 94 ++++++++++++++++++++++++++++++++ tests/test_engine_rate_reset.py | 67 +++++++++++++++++++---- tests/test_inhibition.py | 87 ++++++++++++++++++------------ tests/test_pathways.py | 40 +++++--------- 9 files changed, 428 insertions(+), 144 deletions(-) create mode 100644 src/neural/cell_types.py create mode 100644 src/night/neurons.py create mode 100644 tests/test_cell_types.py diff --git a/src/config.py b/src/config.py index a5db155..3c37f37 100644 --- a/src/config.py +++ b/src/config.py @@ -8,6 +8,11 @@ CONNECTIVITY_PARQUET = os.path.join(DATA_DIR, '2025_Connectivity_783.parquet') COMPLETENESS_CSV = os.path.join(DATA_DIR, '2025_Completeness_783.csv') +ANNOTATIONS_TSV = os.path.join( + os.path.dirname(PROJECT_ROOT), 'flywire_annotations', 'supplemental_files', + 'Supplemental_file1_neuron_annotations.tsv', +) +CELL_TYPE_CACHE = os.path.join(DATA_DIR, 'cell_types_783.parquet') @dataclass(frozen=True) @@ -168,38 +173,54 @@ class DoorDynamics: reopen_settle_sec: float = 0.5 +@dataclass(frozen=True) +class CameraButton: + name: str + x: int + y: int + side: str + channel: str + pursue_to: str | None = None + + +@dataclass(frozen=True) +class TabletVision: + feed_x: int = 470 + feed_y: int = 330 + feed_size: int = 360 + feed_pixels: int = 64 + capture_delay_sec: float = 0.03 + map_ready_sec: float = 0.4 + raise_settle_sec: float = 1.2 + switch_settle_sec: float = 0.5 + reference_sweep_sec: float = 3.0 + reference_frames: int = 24 + noise_margin: float = 1.5 + figure_span_mse: float = 1000.0 + loom_span_mse: float = 1000.0 + loom_speed_span_mse_per_sec: float = 3000.0 + loom_speed_decay_sec: float = 0.3 + lower_settle_sec: float = 0.35 + first_camera: str = '1C' + cameras: tuple = ( + CameraButton('1C', 925, 484, 'left', 'figure', pursue_to='2A'), + CameraButton('2A', 983, 599, 'left', 'loom'), + ) + + +@dataclass(frozen=True) +class CellPopulations: + eye: tuple = ('LPLC2', 'LC4') + figure: tuple = ('LC9', 'LC31a') + loom_size: tuple = ('LPLC2',) + loom_speed: tuple = ('LC4',) + giant_fiber: tuple = ('DNp01',) + looming_escape: tuple = ('DNp04',) + explore: tuple = ('DNp09',) + + @dataclass(frozen=True) class SensoryNeurons: - left_eye_cluster: List[int] = field(default_factory=lambda: [ - 720575940631380695, 720575940639109795, 720575940635307639, 720575940626414279, - 720575940623262461, 720575940641552347, 720575940613145258, 720575940641704656, - 720575940638626118, 720575940632129848, 720575940623652083, 720575940635516084, - 720575940639884878, 720575940610226645, 720575940619263937, 720575940625216202, - 720575940623925774, 720575940615332502, 720575940627685765, 720575940638835288, - 720575940620306904, 720575940640302389, 720575940613763759, 720575940625695269, - 720575940603989420, 720575940628847164, 720575940615953302, 720575940627902078, - 720575940612560680, 720575940642414837, 720575940622680088, 720575940623551684, - 720575940626236041, 720575940618927892, 720575940648418425, 720575940619197318, - 720575940612766691, 720575940645407268, 720575940643740964, 720575940636295019, - 720575940614738444, 720575940628552247, 720575940624208824, 720575940626617084, - 720575940635234478, 720575940616008854, 720575940613219474, 720575940629368015, - 720575940622425271, 720575940629292785, - ]) - right_eye_cluster: List[int] = field(default_factory=lambda: [ - 720575940626745392, 720575940626447848, 720575940625835278, 720575940612045352, - 720575940605825666, 720575940626735016, 720575940618172509, 720575940635040357, - 720575940613804906, 720575940620575476, 720575940612462307, 720575940633552685, - 720575940624014524, 720575940615920453, 720575940616116697, 720575940629967703, - 720575940622992205, 720575940606217138, 720575940618538641, 720575940624919077, - 720575940623047629, 720575940615967377, 720575940610116757, 720575940625001735, - 720575940612254701, 720575940640510427, 720575940635572592, 720575940633588320, - 720575940609592674, 720575940624016778, 720575940626161930, 720575940616735489, - 720575940620001718, 720575940628039034, 720575940612944382, 720575940629384271, - 720575940616166685, 720575940606855554, 720575940632512226, 720575940624611002, - 720575940620783180, 720575940622769159, 720575940614067167, 720575940647358243, - 720575940614526974, 720575940645699124, 720575940613821290, 720575940624931530, - 720575940631397549, 720575940645180196, - ]) camera_inhibitor_left: List[int] = field(default_factory=lambda: [ 720575940635119723, 720575940627096957, 720575940617948445, 720575940610063918, 720575940617564246, 720575940618243009, 720575940606218528, 720575940612710883, @@ -232,17 +253,6 @@ class SensoryNeurons: ]) -@dataclass(frozen=True) -class MotorNeurons: - dnp01_giant_fiber: List[int] = field(default_factory=lambda: [ - 720575940622838154, - 720575940632499757, - ]) - dnp09_explore: List[int] = field(default_factory=lambda: [ - 720575940635872101, - 720575940627652358, - ]) - VISION_CALIBRATION = VisionCalibration() MOTOR_CALIBRATION = MotorCalibration() SIMULATION_PARAMS = SimulationParams() @@ -256,4 +266,5 @@ class MotorNeurons: SEARCH_DYNAMICS = SearchDynamics() DOOR_DYNAMICS = DoorDynamics() SENSORY_NEURONS = SensoryNeurons() -MOTOR_NEURONS = MotorNeurons() \ No newline at end of file +CELL_POPULATIONS = CellPopulations() +TABLET_VISION = TabletVision() \ No newline at end of file diff --git a/src/neural/cell_types.py b/src/neural/cell_types.py new file mode 100644 index 0000000..c6d908a --- /dev/null +++ b/src/neural/cell_types.py @@ -0,0 +1,59 @@ +import logging +import os + +import pandas as pd + +import config + +_log = logging.getLogger(__name__) + +COLUMNS = ['root_id', 'cell_type', 'side'] + + +class CellTypeIndex: + def __init__(self, table: pd.DataFrame): + self._table = table[COLUMNS].dropna(subset=['cell_type']) + + @classmethod + def load(cls, annotations: str = config.ANNOTATIONS_TSV, + cache: str = config.CELL_TYPE_CACHE) -> 'CellTypeIndex': + if _cache_is_current(cache, annotations): + return cls(pd.read_parquet(cache)) + if not os.path.isfile(annotations): + raise FileNotFoundError( + 'the FlyWire annotation table is missing at %s, run src/scripts/fetch_data.py' + % annotations + ) + _log.info('indexing FlyWire cell types from %s', os.path.basename(annotations)) + table = pd.read_csv(annotations, sep='\t', usecols=COLUMNS, dtype={'cell_type': str, 'side': str}) + table['root_id'] = table['root_id'].astype('int64') + _write_cache(table, cache) + return cls(table) + + def root_ids(self, cell_types, side: str | None = None) -> list: + rows = self._table[self._table['cell_type'].isin(tuple(cell_types))] + if side is not None: + rows = rows[rows['side'] == side] + return sorted(int(root_id) for root_id in rows['root_id']) + + def describe(self, root_id: int) -> str: + rows = self._table[self._table['root_id'] == root_id] + if rows.empty: + return f'untyped {root_id}' + row = rows.iloc[0] + return f'{row["cell_type"]} ({row["side"]}) {root_id}' + + +def _cache_is_current(cache: str, annotations: str) -> bool: + if not os.path.isfile(cache): + return False + if not os.path.isfile(annotations): + return True + return os.path.getmtime(cache) >= os.path.getmtime(annotations) + + +def _write_cache(table: pd.DataFrame, cache: str): + try: + table[COLUMNS].to_parquet(cache) + except OSError as exc: + _log.warning('cell type cache not written, it will be rebuilt next run: %s', exc) diff --git a/src/night/engine.py b/src/night/engine.py index ac7f705..4c7b365 100644 --- a/src/night/engine.py +++ b/src/night/engine.py @@ -2,11 +2,13 @@ import config from brain_adapter import BrainAdapter +from neural.cell_types import CellTypeIndex +from night.neurons import build_neuron_map, stimulus_rates from night.state import SensoryState class ConnectomeEngine: - def __init__(self, device: str): + def __init__(self, device: str, cell_index: CellTypeIndex | None = None): self._device = device self._adapter = BrainAdapter( config.COMPLETENESS_CSV, @@ -14,21 +16,10 @@ def __init__(self, device: str): config.DATA_DIR, device, ) - sensory = config.SENSORY_NEURONS - motor = config.MOTOR_NEURONS - - resolve = self._adapter.map_neuron_ids_to_indices - self._l_sensory_idx = resolve(sensory.left_eye_cluster, 'left eye cluster') - self._r_sensory_idx = resolve(sensory.right_eye_cluster, 'right eye cluster') - all_sensory = self._l_sensory_idx + self._r_sensory_idx - - self._l_motor_idx = resolve([motor.dnp01_giant_fiber[0]], 'DNp01 left giant fiber') - self._r_motor_idx = resolve([motor.dnp01_giant_fiber[1]], 'DNp01 right giant fiber') - self._explore_idx = resolve(motor.dnp09_explore, 'DNp09 explore') - self._l_inhib_idx = resolve(sensory.camera_inhibitor_left, 'left camera inhibitors') - self._r_inhib_idx = resolve(sensory.camera_inhibitor_right, 'right camera inhibitors') - - self._adapter.initialize_model(exc_indices=all_sensory) + self.neurons = build_neuron_map( + cell_index or CellTypeIndex.load(), self._adapter.map_neuron_ids_to_indices, + ) + self._adapter.initialize_model(exc_indices=self.neurons.stimulus_indices()) self._rates = torch.zeros(1, self._adapter.num_neurons, device=device) self._base_rate = config.SIMULATION_PARAMS.base_sensory_rate_hz @@ -41,7 +32,7 @@ def __init__(self, device: str): @property def sensory_span(self) -> int: - return len(self._l_sensory_idx) * self._steps + return max(len(self.l_sensory_idx), len(self.r_sensory_idx)) * self._steps def step(self, state: SensoryState, current_time: float) -> torch.Tensor: self.frames += 1 @@ -49,25 +40,28 @@ def step(self, state: SensoryState, current_time: float) -> torch.Tensor: self.driven_frames['left'] += 1 if state.right_rate > 0.0: self.driven_frames['right'] += 1 - self._rates.zero_() - - self._rates[:, self._l_sensory_idx] = self._base_rate * state.left_rate - self._rates[:, self._r_sensory_idx] = self._base_rate * state.right_rate - self._rates[:, self._l_inhib_idx] = state.cam_inhib - self._rates[:, self._r_inhib_idx] = state.cam_inhib - - noise_hz = config.FORAGING_PARAMS.subliminal_noise_hz - self._rates += torch.rand_like(self._rates) * noise_hz + self._load_rates(state) spikes = self._adapter.step(self._rates, steps=self._steps) v = self._adapter.membrane_potential() self.eye_membrane_diff = float( - v[0, self._l_sensory_idx].mean() - v[0, self._r_sensory_idx].mean() + v[0, self.l_sensory_idx].mean() - v[0, self.r_sensory_idx].mean() ) - self.explore_membrane = float(v[0, self._explore_idx].mean()) + self.explore_membrane = float(v[0, self.explore_idx].mean()) return spikes + def reset(self): + self._adapter.model.reset_state() + + def _load_rates(self, state: SensoryState): + self._rates.zero_() + for indices, rate_hz in stimulus_rates(self.neurons, state, self._base_rate): + if rate_hz > 0.0: + self._rates[:, indices] = torch.clamp(self._rates[:, indices], min=rate_hz) + noise_hz = config.FORAGING_PARAMS.subliminal_noise_hz + self._rates += torch.rand_like(self._rates) * noise_hz + @property def num_neurons(self) -> int: return self._adapter.num_neurons @@ -78,20 +72,29 @@ def synapses(self): @property def l_motor_idx(self): - return self._l_motor_idx + return self.neurons.outputs['giant_fiber_left'] @property def r_motor_idx(self): - return self._r_motor_idx + return self.neurons.outputs['giant_fiber_right'] + + @property + def l_escape_idx(self): + return self.neurons.outputs['looming_escape_left'] + + @property + def r_escape_idx(self): + return self.neurons.outputs['looming_escape_right'] @property def explore_idx(self): - return self._explore_idx + return self.neurons.outputs['explore'] @property def l_sensory_idx(self): - return self._l_sensory_idx + return self.neurons.inputs['eye_left'] @property def r_sensory_idx(self): - return self._r_sensory_idx + return self.neurons.inputs['eye_right'] + diff --git a/src/night/neurons.py b/src/night/neurons.py new file mode 100644 index 0000000..e2b1756 --- /dev/null +++ b/src/night/neurons.py @@ -0,0 +1,55 @@ +from dataclasses import dataclass + +import config + +SIDES = ('left', 'right') +STIMULUS_ROLES = ('eye', 'figure', 'loom_size', 'loom_speed') +READOUT_ROLES = ('giant_fiber', 'looming_escape') + + +@dataclass(frozen=True) +class NeuronMap: + inputs: dict + outputs: dict + + def stimulus_indices(self) -> list: + chosen = set() + for role in STIMULUS_ROLES: + for side in SIDES: + chosen.update(self.inputs.get(f'{role}_{side}', ())) + return sorted(chosen) + + def sizes(self) -> dict: + return {name: len(indices) for name, indices in {**self.inputs, **self.outputs}.items()} + + +def build_neuron_map(cell_index, to_indices) -> NeuronMap: + roles = config.CELL_POPULATIONS + inhibitors = config.SENSORY_NEURONS + inputs, outputs = {}, {} + + for side in SIDES: + for role in STIMULUS_ROLES: + types = getattr(roles, role) + inputs[f'{role}_{side}'] = to_indices( + cell_index.root_ids(types, side), f'{side} {"+".join(types)}', + ) + for role in READOUT_ROLES: + types = getattr(roles, role) + outputs[f'{role}_{side}'] = to_indices( + cell_index.root_ids(types, side), f'{side} {"+".join(types)}', + ) + + inputs['inhibitors_left'] = to_indices(inhibitors.camera_inhibitor_left, 'left camera inhibitors') + inputs['inhibitors_right'] = to_indices(inhibitors.camera_inhibitor_right, 'right camera inhibitors') + outputs['explore'] = to_indices(cell_index.root_ids(roles.explore), '+'.join(roles.explore)) + return NeuronMap(inputs, outputs) + + +def stimulus_rates(neurons: NeuronMap, state, base_rate_hz: float): + yield neurons.inputs['eye_left'], base_rate_hz * state.left_rate + yield neurons.inputs['eye_right'], base_rate_hz * state.right_rate + yield neurons.inputs['inhibitors_left'], state.cam_inhib + yield neurons.inputs['inhibitors_right'], state.cam_inhib + for population, level in state.tablet_drive.items(): + yield neurons.inputs[population], base_rate_hz * level diff --git a/src/night/state.py b/src/night/state.py index 00b8dd2..754a1b0 100644 --- a/src/night/state.py +++ b/src/night/state.py @@ -11,11 +11,17 @@ class SensoryState: camera_open: bool = False l_spike: bool = False r_spike: bool = False + l_escape: bool = False + r_escape: bool = False + explore_spike: bool = False l_sensory_count: int = 0 r_sensory_count: int = 0 forage_bias: float = 0.0 office_centred: bool = True blind_until: dict = field(default_factory=lambda: {'left': 0.0, 'right': 0.0}) + tablet_camera: str | None = None + tablet_readable_at: float = 0.0 + tablet_drive: dict = field(default_factory=dict) @dataclass diff --git a/tests/test_cell_types.py b/tests/test_cell_types.py new file mode 100644 index 0000000..c135647 --- /dev/null +++ b/tests/test_cell_types.py @@ -0,0 +1,94 @@ +import os +import sys +import tempfile + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) + +import pandas as pd + +import config +from neural.cell_types import CellTypeIndex +from night.neurons import NeuronMap, build_neuron_map, stimulus_rates +from night.state import SensoryState + +TABLE = pd.DataFrame({ + 'root_id': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], + 'cell_type': ['LPLC2', 'LPLC2', 'LC4', 'LC4', 'LC9', 'LC31a', 'DNp01', 'DNp01', + 'DNp04', 'DNp04', 'DNp09', 'DNp09', None], + 'side': ['left', 'right', 'left', 'right', 'left', 'left', 'left', 'right', + 'left', 'right', 'left', 'right', 'left'], +}) + +LOOKUP_CASES = ( + (('LPLC2',), 'left', [1]), + (('LPLC2', 'LC4'), 'right', [2, 4]), + (('DNp09',), None, [11, 12]), + (('LC9', 'LC31a'), 'right', []), + (('NOPE',), 'left', []), +) + + +def test_root_ids_select_by_type_and_side(): + index = CellTypeIndex(TABLE) + for types, side, expected in LOOKUP_CASES: + assert index.root_ids(types, side) == expected, (types, side) + + +def test_describe_names_the_type_and_side(): + index = CellTypeIndex(TABLE) + assert index.describe(7) == 'DNp01 (left) 7' + assert index.describe(999) == 'untyped 999' + + +def test_the_cache_is_reused_until_the_annotations_change(): + with tempfile.TemporaryDirectory() as folder: + tsv = os.path.join(folder, 'annotations.tsv') + cache = os.path.join(folder, 'cache.parquet') + TABLE.assign(extra='x').to_csv(tsv, sep='\t', index=False) + + first = CellTypeIndex.load(tsv, cache) + assert os.path.isfile(cache) + os.remove(tsv) + second = CellTypeIndex.load(tsv, cache) + assert first.root_ids(['LC4'], 'left') == second.root_ids(['LC4'], 'left') == [3] + + +def _identity(root_ids, label): + return list(root_ids) + + +def test_the_neuron_map_resolves_every_role_from_cell_types(): + neurons = build_neuron_map(CellTypeIndex(TABLE), _identity) + assert neurons.inputs['eye_left'] == [1, 3] + assert neurons.inputs['figure_left'] == [5, 6] + assert neurons.inputs['loom_size_left'] == [1] + assert neurons.inputs['loom_speed_right'] == [4] + assert neurons.outputs['giant_fiber_right'] == [8] + assert neurons.outputs['looming_escape_left'] == [9] + assert neurons.outputs['explore'] == [11, 12] + assert neurons.inputs['inhibitors_left'] == list(config.SENSORY_NEURONS.camera_inhibitor_left) + + +def test_inhibitors_keep_their_refractory_period(): + neurons = build_neuron_map(CellTypeIndex(TABLE), _identity) + stimulated = set(neurons.stimulus_indices()) + assert {1, 2, 3, 4, 5, 6} <= stimulated + assert not stimulated & set(config.SENSORY_NEURONS.camera_inhibitor_left) + + +def test_stimulus_rates_scale_levels_and_pass_the_inhibitor_rate_through(): + neurons = NeuronMap( + inputs={'eye_left': [0], 'eye_right': [1], 'inhibitors_left': [2], + 'inhibitors_right': [3], 'figure_left': [4]}, + outputs={}, + ) + state = SensoryState(left_rate=1.0, cam_inhib=250.0, tablet_drive={'figure_left': 0.5}) + rates = [(tuple(indices), rate) for indices, rate in stimulus_rates(neurons, state, 1000.0)] + assert rates == [((0,), 1000.0), ((1,), 0.0), ((2,), 250.0), ((3,), 250.0), ((4,), 500.0)] + + +if __name__ == '__main__': + for name, fn in sorted(globals().items()): + if name.startswith('test_'): + fn() + print('ok') diff --git a/tests/test_engine_rate_reset.py b/tests/test_engine_rate_reset.py index ddaa8cd..fa9c155 100644 --- a/tests/test_engine_rate_reset.py +++ b/tests/test_engine_rate_reset.py @@ -8,23 +8,41 @@ import config from night.engine import ConnectomeEngine +from night.neurons import NeuronMap from night.state import SensoryState +NEURONS = 12 +TOUCHED = set(range(8)) + + +def _neuron_map() -> NeuronMap: + return NeuronMap( + inputs={ + 'eye_left': [0, 5], 'eye_right': [1], + 'inhibitors_left': [2], 'inhibitors_right': [3], + 'figure_left': [6], 'figure_right': [], + 'loom_size_left': [0], 'loom_size_right': [], + 'loom_speed_left': [5], 'loom_speed_right': [], + }, + outputs={ + 'giant_fiber_left': [7], 'giant_fiber_right': [8], + 'looming_escape_left': [9], 'looming_escape_right': [10], + 'explore': [4], + }, + ) + -def _make_bare_engine(num_neurons: int = 10) -> ConnectomeEngine: +def _make_bare_engine() -> ConnectomeEngine: engine = ConnectomeEngine.__new__(ConnectomeEngine) - engine._l_sensory_idx = [0] - engine._r_sensory_idx = [1] - engine._l_inhib_idx = [2] - engine._r_inhib_idx = [3] - engine._explore_idx = [4] - engine._rates = torch.zeros(1, num_neurons) + engine.neurons = _neuron_map() + engine._rates = torch.zeros(1, NEURONS) engine._base_rate = 1000.0 engine._steps = 2 engine.frames = 0 engine.driven_frames = {'left': 0, 'right': 0} engine._adapter = MagicMock() engine._adapter.step.side_effect = lambda rates, steps: rates.clone() + engine._adapter.membrane_potential.return_value = torch.zeros(1, NEURONS) return engine @@ -35,7 +53,7 @@ def test_untouched_neurons_do_not_accumulate_noise_across_frames(): for frame in range(500): engine.step(state, current_time=frame * 0.1) - untouched = [i for i in range(10) if i not in (0, 1, 2, 3, 4)] + untouched = [i for i in range(NEURONS) if i not in TOUCHED] max_untouched_rate = engine._rates[:, untouched].max().item() subliminal_noise_hz = config.FORAGING_PARAMS.subliminal_noise_hz @@ -58,7 +76,36 @@ def test_assigned_clusters_reflect_only_the_current_frame_state(): assert low_left < engine._base_rate +TABLET_CASES = ( + ('figure_left', 6, 0.25), + ('loom_size_left', 0, 0.5), + ('loom_speed_left', 5, 1.0), +) + + +def test_tablet_drive_reaches_only_its_population_for_one_frame(): + for population, index, level in TABLET_CASES: + engine = _make_bare_engine() + + engine.step(SensoryState(tablet_drive={population: level}), current_time=0.0) + driven = engine._rates[0, index].item() + engine.step(SensoryState(), current_time=0.1) + + assert driven >= engine._base_rate * level, population + assert engine._rates[0, index].item() < config.FORAGING_PARAMS.subliminal_noise_hz, population + + +def test_overlapping_populations_keep_the_stronger_drive(): + engine = _make_bare_engine() + + state = SensoryState(left_rate=1.0, tablet_drive={'loom_size_left': 0.2}) + engine.step(state, current_time=0.0) + + assert engine._rates[0, 0].item() >= engine._base_rate + + if __name__ == '__main__': - test_untouched_neurons_do_not_accumulate_noise_across_frames() - test_assigned_clusters_reflect_only_the_current_frame_state() + for name, fn in sorted(globals().items()): + if name.startswith('test_'): + fn() print('ok') diff --git a/tests/test_inhibition.py b/tests/test_inhibition.py index aaeeeac..9970d33 100644 --- a/tests/test_inhibition.py +++ b/tests/test_inhibition.py @@ -1,49 +1,70 @@ import sys import os -import torch sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) + import config -from neural.models import FlyBrainModel -from neural.data_loader import get_hash_tables, load_connectome_weights +from night.engine import ConnectomeEngine +from night.state import SensoryState +FRAMES = 2500 +TABLET_FRAMES = 40 +TABLET_UP_HZ = config.SIMULATION_PARAMS.base_sensory_rate_hz -def test_biological_inhibition(): - device = 'cpu' - flyid2i, _ = get_hash_tables(config.COMPLETENESS_CSV) - weights = load_connectome_weights( - config.CONNECTIVITY_PARQUET, config.COMPLETENESS_CSV, config.DATA_DIR, - csr=True, device=device, - ) - num_neurons = weights.shape[0] - weights = weights * config.SIMULATION_PARAMS.arousal_multiplier +_ENGINE = {} - exc_indices = [flyid2i[n] for n in config.SENSORY_NEURONS.left_eye_cluster if n in flyid2i] - inhib_indices = [flyid2i[n] for n in config.SENSORY_NEURONS.camera_inhibitor_left if n in flyid2i] - motor_idx = flyid2i[config.MOTOR_NEURONS.dnp01_giant_fiber[0]] - model = FlyBrainModel(num_neurons, weights, exc_indices=exc_indices + inhib_indices, device=device) +def _engine() -> ConnectomeEngine: + if 'engine' not in _ENGINE: + _ENGINE['engine'] = ConnectomeEngine('cpu') + return _ENGINE['engine'] - rates = torch.zeros(1, num_neurons, device=device) - rates[:, exc_indices] = config.SIMULATION_PARAMS.base_sensory_rate_hz - total_spikes = sum( - model.step(rates)[0, motor_idx].item() - for _ in range(5000) - ) - assert total_spikes > 0, f'expected startle reflex, got {total_spikes} spikes' - model.reset_state() +def _spike_frames(state: SensoryState, frames: int) -> dict: + engine = _engine() + engine.reset() + readouts = { + 'giant_fiber': engine.l_motor_idx, + 'looming_escape': engine.l_escape_idx, + 'explore': engine.explore_idx, + } + counts = dict.fromkeys(readouts, 0) + for frame in range(frames): + spikes = engine.step(state, frame * engine.frame_dt)[0] + for name, indices in readouts.items(): + counts[name] += int(bool(spikes[indices].any())) + return counts - rates[:, inhib_indices] = config.SIMULATION_PARAMS.base_sensory_rate_hz - total_inhibited = sum( - model.step(rates)[0, motor_idx].item() - for _ in range(5000) - ) - assert total_inhibited == 0, ( - f'inhibition failed, giant fiber spiked {total_inhibited} times with camera open' + +def test_biological_inhibition(): + startle = _spike_frames(SensoryState(left_rate=1.0), FRAMES) + assert startle['giant_fiber'] > 0, f'expected startle reflex, got {startle}' + + inhibited = _spike_frames(SensoryState(left_rate=1.0, cam_inhib=TABLET_UP_HZ), FRAMES) + assert inhibited['giant_fiber'] == 0, ( + f'inhibition failed, giant fiber spiked on {inhibited["giant_fiber"]} frames with camera open' ) +TABLET_CASES = ( + ('looming on the camera', {'loom_size_left': 0.25, 'loom_speed_left': 0.25}, + 'looming_escape', ('giant_fiber', 'explore')), + ('a figure in the cove', {'figure_left': 0.25}, + 'explore', ('giant_fiber', 'looming_escape')), +) + + +def test_each_tablet_channel_wakes_its_own_descending_neuron(): + for label, drive, answers, silent in TABLET_CASES: + counts = _spike_frames(SensoryState(cam_inhib=TABLET_UP_HZ, tablet_drive=drive), TABLET_FRAMES) + assert counts[answers] > 0, f'{label}: {answers} never fired, {counts}' + assert counts['giant_fiber'] == 0, f'{label}: the giant fiber broke through the inhibitors, {counts}' + crossed = [name for name in silent if name != 'giant_fiber' and counts[name] > counts[answers] / 4] + assert not crossed, f'{label}: {crossed} answered a channel that is not theirs, {counts}' + + if __name__ == '__main__': - test_biological_inhibition() - print('ok') \ No newline at end of file + for name, fn in sorted(globals().items()): + if name.startswith('test_'): + fn() + print('ok') diff --git a/tests/test_pathways.py b/tests/test_pathways.py index 482efdd..10938b2 100644 --- a/tests/test_pathways.py +++ b/tests/test_pathways.py @@ -4,42 +4,30 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) import torch -import config -from neural.models import FlyBrainModel -from neural.data_loader import get_hash_tables, load_connectome_weights +from night.engine import ConnectomeEngine +from night.state import SensoryState +FRAMES = 2500 -def test_sensory_to_motor_pathway(): - device = 'cpu' - flyid2i, i2flyid = get_hash_tables(config.COMPLETENESS_CSV) - weights = load_connectome_weights( - config.CONNECTIVITY_PARQUET, config.COMPLETENESS_CSV, config.DATA_DIR, - csr=True, device=device, - ) - num_neurons = weights.shape[0] - weights = weights * config.SIMULATION_PARAMS.arousal_multiplier - - exc_indices = [flyid2i[n] for n in config.SENSORY_NEURONS.left_eye_cluster if n in flyid2i] - motor_idx = flyid2i[config.MOTOR_NEURONS.dnp01_giant_fiber[0]] - model = FlyBrainModel(num_neurons, weights, exc_indices=exc_indices, device=device) - - rates = torch.zeros(1, num_neurons, device=device) - rates[:, exc_indices] = config.SIMULATION_PARAMS.base_sensory_rate_hz +def test_sensory_to_motor_pathway(): + engine = ConnectomeEngine('cpu') + state = SensoryState(left_rate=1.0) + eye = set(engine.l_sensory_idx) - total_spikes_per_neuron = torch.zeros(num_neurons, device=device) - for _ in range(5000): - total_spikes_per_neuron += model.step(rates)[0] + total_spikes_per_neuron = torch.zeros(engine.num_neurons) + for frame in range(FRAMES): + total_spikes_per_neuron += engine.step(state, frame * engine.frame_dt)[0] active_neurons = (total_spikes_per_neuron > 0).nonzero(as_tuple=True)[0] - downstream_active = [idx.item() for idx in active_neurons if idx.item() not in exc_indices] + downstream_active = [idx.item() for idx in active_neurons if idx.item() not in eye] assert len(downstream_active) > 0, 'no interneurons recruited from sensory input' - assert total_spikes_per_neuron[motor_idx].item() > 0, ( - f'signal did not reach motor neuron {i2flyid[motor_idx]}' + assert total_spikes_per_neuron[engine.l_motor_idx].sum().item() > 0, ( + 'signal did not reach the left giant fiber' ) if __name__ == '__main__': test_sensory_to_motor_pathway() - print('ok') \ No newline at end of file + print('ok') From 502cb05afd949640683e61eaeeeebd4bcd339a23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Artur=20Mendon=C3=A7a=20Arruda?= Date: Thu, 1 Oct 2026 01:47:59 -0300 Subject: [PATCH 02/14] feat(env): capture the tablet feed and click the camera map --- src/env/input_controller.py | 21 +++++- src/env/overlay.py | 5 +- src/env/tablet_feed.py | 111 ++++++++++++++++++++++++++++++++ tests/test_hallway_detection.py | 1 + tests/test_overlay_placement.py | 2 +- 5 files changed, 137 insertions(+), 3 deletions(-) create mode 100644 src/env/tablet_feed.py diff --git a/src/env/input_controller.py b/src/env/input_controller.py index 7903eb6..7349367 100644 --- a/src/env/input_controller.py +++ b/src/env/input_controller.py @@ -123,6 +123,14 @@ def _dispatch(cmd, action, _reach, _click, _slide, light_state, camera_state, do _slide(motor.camera_hover_x, motor.camera_hover_y, motor.screen_center_y) camera_state['open'] = False + elif action == 'select_camera': + button = _camera_button(cmd.get('camera')) + if button is not None: + time.sleep(cmd.get('settle_sec', 0.0)) + desktop.move_cursor(*anchor.point(button.x, button.y)) + time.sleep(motor.click_delay_sec) + _click() + elif action == 'nudge_camera_bar': _slide(motor.camera_hover_x, motor.screen_center_y, motor.camera_hover_y) time.sleep(motor.camera_bar_dwell_sec) @@ -133,6 +141,14 @@ def _dispatch(cmd, action, _reach, _click, _slide, light_state, camera_state, do _log.warning('unknown command action: %s', action) +def _camera_button(name): + for button in config.TABLET_VISION.cameras: + if button.name == name: + return button + _log.warning('no map button is calibrated for camera %s', name) + return None + + def start_worker(): t = threading.Thread(target=_worker, daemon=True) t.start() @@ -177,4 +193,7 @@ def close_camera(self, force: bool = False) -> threading.Event: return _submit(action='close_camera', force=force) def nudge_camera_bar(self) -> threading.Event: - return _submit(action='nudge_camera_bar') \ No newline at end of file + return _submit(action='nudge_camera_bar') + + def select_camera(self, camera: str, settle_sec: float = 0.0) -> threading.Event: + return _submit(action='select_camera', camera=camera, settle_sec=settle_sec) diff --git a/src/env/overlay.py b/src/env/overlay.py index ad1e341..ede6f23 100644 --- a/src/env/overlay.py +++ b/src/env/overlay.py @@ -27,10 +27,13 @@ def capture_regions() -> list: left = anchor.point(vision.left_target_x, vision.left_target_y) right = anchor.point(vision.right_target_x, vision.right_target_y) patch = anchor.point(camera.patch_x, camera.patch_y) + tablet = config.TABLET_VISION + feed = anchor.point(tablet.feed_x, tablet.feed_y) return [ centered_rect(left[0], left[1], anchor.size(vision.left_bbox_size)), centered_rect(right[0], right[1], anchor.size(vision.right_bbox_size)), centered_rect(patch[0], patch[1], anchor.size(camera.patch_size)), + centered_rect(feed[0], feed[1], anchor.size(tablet.feed_size)), ] @@ -43,7 +46,7 @@ def motor_regions(pad: int = 40) -> list: (motor.right_light_button_x, motor.right_light_button_y), (motor.camera_hover_x, motor.camera_hover_y), (motor.camera_hover_x, motor.screen_center_y), - ] + ] + [(button.x, button.y) for button in config.TABLET_VISION.cameras] reach = anchor.size(pad * 2) return [centered_rect(*anchor.point(x, y), reach) for x, y in targets] diff --git a/src/env/tablet_feed.py b/src/env/tablet_feed.py new file mode 100644 index 0000000..682aa0f --- /dev/null +++ b/src/env/tablet_feed.py @@ -0,0 +1,111 @@ +import collections +import logging +import threading +import time + +import cv2 +import mss +import numpy as np + +import config +from env import anchor + +_log = logging.getLogger(__name__) + + +def prepare(frame: np.ndarray, pixels: int) -> np.ndarray: + gray = frame if frame.ndim == 2 else cv2.cvtColor(frame, cv2.COLOR_BGRA2GRAY) + small = cv2.resize(gray.astype(np.float32), (pixels, pixels), interpolation=cv2.INTER_AREA) + return cv2.GaussianBlur(small, (3, 3), 0) + + +def bank_distance(frame: np.ndarray, bank: list) -> float: + if not bank: + return 0.0 + return min(float(np.mean((frame - reference) ** 2)) for reference in bank) + + +def bank_noise(bank: list) -> float: + if len(bank) < 2: + return 0.0 + return max(bank_distance(frame, bank[:i] + bank[i + 1:]) for i, frame in enumerate(bank)) + + +def spread_sample(frames: list, count: int) -> list: + if len(frames) <= count: + return list(frames) + step = len(frames) / count + return [frames[int(i * step)] for i in range(count)] + + +class TabletFeed: + def __init__(self, params=None): + self._p = params or config.TABLET_VISION + self._latest: collections.deque = collections.deque(maxlen=1) + self._lock = threading.Lock() + self._active = threading.Event() + self._banks: dict = {} + self._noise: dict = {} + threading.Thread(target=self._capture_loop, daemon=True).start() + + def activate(self): + self._active.set() + + def deactivate(self): + self._active.clear() + with self._lock: + self._latest.clear() + + def has_reference(self, camera: str) -> bool: + return bool(self._banks.get(camera)) + + def noise(self, camera: str) -> float: + return self._noise.get(camera, 0.0) + + def contrast(self, camera: str) -> float | None: + bank = self._banks.get(camera) + with self._lock: + frame = self._latest[-1] if self._latest else None + if not bank or frame is None: + return None + return bank_distance(frame, bank) + + def capture_reference(self, camera: str) -> bool: + frames = self._collect(self._p.reference_sweep_sec) + bank = spread_sample(frames, self._p.reference_frames) + if not bank: + _log.warning('no frames arrived from the tablet while recording camera %s', camera) + return False + self._banks[camera] = bank + self._noise[camera] = bank_noise(bank) * self._p.noise_margin + _log.info('camera %s reference: %d frames, noise floor %.0f mse', + camera, len(bank), self._noise[camera]) + return True + + def _collect(self, seconds: float) -> list: + frames, deadline, last = [], time.time() + seconds, None + while time.time() < deadline: + with self._lock: + frame = self._latest[-1] if self._latest else None + if frame is not None and frame is not last: + frames.append(frame) + last = frame + time.sleep(self._p.capture_delay_sec) + return frames + + def _capture_loop(self): + with mss.MSS() as screen: + while True: + self._active.wait() + frame = self._grab(screen) + with self._lock: + if self._active.is_set(): + self._latest.append(frame) + time.sleep(self._p.capture_delay_sec) + + def _grab(self, screen) -> np.ndarray: + centre_x, centre_y = anchor.point(self._p.feed_x, self._p.feed_y) + size = anchor.size(self._p.feed_size) + box = {'top': max(0, centre_y - size // 2), 'left': max(0, centre_x - size // 2), + 'width': size, 'height': size} + return prepare(np.array(screen.grab(box)), self._p.feed_pixels) diff --git a/tests/test_hallway_detection.py b/tests/test_hallway_detection.py index d68da12..53b4448 100644 --- a/tests/test_hallway_detection.py +++ b/tests/test_hallway_detection.py @@ -18,6 +18,7 @@ def _eye(frames): vision._buf_lock = threading.Lock() vision.mse_threshold = config.FORAGING_PARAMS.mse_threshold vision._peak_mse = {'left': 0.0, 'right': 0.0} + vision._last_mse = {'left': 0.0, 'right': 0.0} vision._threat_written_left = True vision._threat_written_right = True vision.ref_left = np.zeros((SIZE, SIZE), dtype=np.float32) diff --git a/tests/test_overlay_placement.py b/tests/test_overlay_placement.py index 46839a7..7f43c97 100644 --- a/tests/test_overlay_placement.py +++ b/tests/test_overlay_placement.py @@ -91,7 +91,7 @@ def test_real_config_regions_leave_room_on_this_layout(): regions = capture_regions() vision = config.VISION_CALIBRATION - assert len(regions) == 3 + assert len(regions) == 4 assert regions[0] == centered_rect( vision.left_target_x, vision.left_target_y, vision.left_bbox_size ) From 78f9b40797972e0d5208ca89b9267b6d637975cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Artur=20Mendon=C3=A7a=20Arruda?= Date: Thu, 1 Oct 2026 01:49:31 -0300 Subject: [PATCH 03/14] feat(night): watch Foxy through the tablet and close the door on DNp04 --- src/main.py | 13 +- src/night/calibration.py | 7 +- src/night/doors.py | 65 ++++++++ src/night/monitor.py | 32 +++- src/night/tablet/__init__.py | 0 src/night/tablet/gaze.py | 37 +++++ src/night/tablet/references.py | 45 +++++ src/night/tablet/senses.py | 39 +++++ src/night/tablet/transduction.py | 44 +++++ src/night/tablet/watch.py | 72 ++++++++ src/night/tasks.py | 267 +++++++++++++++--------------- src/telemetry.py | 42 ++++- tests/test_search_loop_wiring.py | 2 + tests/test_tablet_escape.py | 165 ++++++++++++++++++ tests/test_tablet_gaze.py | 125 ++++++++++++++ tests/test_tablet_senses.py | 71 ++++++++ tests/test_tablet_transduction.py | 108 ++++++++++++ 17 files changed, 996 insertions(+), 138 deletions(-) create mode 100644 src/night/doors.py create mode 100644 src/night/tablet/__init__.py create mode 100644 src/night/tablet/gaze.py create mode 100644 src/night/tablet/references.py create mode 100644 src/night/tablet/senses.py create mode 100644 src/night/tablet/transduction.py create mode 100644 src/night/tablet/watch.py create mode 100644 tests/test_tablet_escape.py create mode 100644 tests/test_tablet_gaze.py create mode 100644 tests/test_tablet_senses.py create mode 100644 tests/test_tablet_transduction.py diff --git a/src/main.py b/src/main.py index 18040be..4636e58 100644 --- a/src/main.py +++ b/src/main.py @@ -9,10 +9,13 @@ from env.cascade import CascadeTracer from env.input_controller import FNAFController, start_worker from env.overlay import anchor_to_game +from env.tablet_feed import TabletFeed from env.vision import FNAFVision from night.engine import ConnectomeEngine from night.panel import brain_view_task from night.state import SaccadeRequest, SensoryState +from night.tablet.senses import TabletSenses +from night.tablet.watch import TabletWatch, build_gaze from night.tasks import engine_task, saccade_task, vision_task from recorder import SessionRecorder from telemetry import ConnectomeTelemetry @@ -37,16 +40,20 @@ async def _run(telemetry: ConnectomeTelemetry, trace: SessionRecorder, begin_nig feed = SpikeFeed(engine.num_neurons, tracer) highlights = {'gaze': '--', 'gf_l': False, 'gf_r': False, 'camera': False} saccade = SaccadeRequest() + tablet_feed = TabletFeed() + tablet = TabletWatch(build_gaze(), tablet_feed, controller, telemetry, state) async with asyncio.TaskGroup() as tg: - tg.create_task(vision_task(vision, controller, state, shutdown)) + tg.create_task(vision_task( + vision, controller, state, shutdown, TabletSenses(tablet_feed), + )) tg.create_task(saccade_task( engine, vision, controller, state, shutdown, telemetry, calibration_done, - saccade, view_ready, begin_night, + saccade, view_ready, begin_night, tablet_feed, )) tg.create_task(engine_task( engine, vision, controller, state, shutdown, telemetry, calibration_done, - feed, highlights, saccade, trace, + feed, highlights, saccade, trace, tablet, )) tg.create_task(brain_view_task( feed, highlights, shutdown, view_ready, vision.latest_patches, diff --git a/src/night/calibration.py b/src/night/calibration.py index 0f9bced..6cdde05 100644 --- a/src/night/calibration.py +++ b/src/night/calibration.py @@ -6,6 +6,7 @@ from env.overlay import anchor_to_game, game_in_front, window_title from env.vision import FNAFVision from night.motor import await_motor +from night.tablet.references import capture_camera_references _log = logging.getLogger(__name__) @@ -33,7 +34,7 @@ async def calibrate_eye_references_live(vision: FNAFVision, controller: FNAFCont vision.save_reference_to_disk() -async def calibrate(vision: FNAFVision, controller: FNAFController, begin_night=None): +async def calibrate(vision: FNAFVision, controller: FNAFController, begin_night=None, tablet_feed=None): settle = config.FORAGING_PARAMS.light_activation_settle_sec if begin_night is None: @@ -52,6 +53,10 @@ async def calibrate(vision: FNAFVision, controller: FNAFController, begin_night= vision.capture_camera_closed_reference() _log.info('office reference captured with the view centred') + if tablet_feed is not None and await capture_camera_references(tablet_feed, controller, vision): + _log.info('camera references captured for %s', + ', '.join(camera.name for camera in config.TABLET_VISION.cameras)) + async def countdown_to_the_night(seconds: float): _log.info('the panel is up. Go to the game and start the night, beginning in %.0fs', seconds) diff --git a/src/night/doors.py b/src/night/doors.py new file mode 100644 index 0000000..f140f22 --- /dev/null +++ b/src/night/doors.py @@ -0,0 +1,65 @@ +import logging + +import config +from night.state import MotorRefrac, SensoryState + +_log = logging.getLogger(__name__) + +SIDES = ('left', 'right') + + +class DoorControl: + def __init__(self, controller, telemetry, state: SensoryState): + dynamics = config.DOOR_DYNAMICS + self._controller = controller + self._telemetry = telemetry + self._state = state + self._hold_leak = dynamics.hold_leak_per_frame + self._release_threshold = dynamics.release_threshold + self._reopen_settle = dynamics.reopen_settle_sec + self._nominal_fps = config.SIMULATION_PARAMS.target_fps + self._motor_sec = config.FORAGING_PARAMS.motor_refractory_sec + self._refrac = MotorRefrac() + self.closed = {side: False for side in SIDES} + self._closed_at = {side: 0.0 for side in SIDES} + self._hold = {side: 0.0 for side in SIDES} + + def refractory(self, side: str, now: float) -> bool: + return now <= getattr(self._refrac, side) + + def on_giant_fiber(self, side: str, now: float) -> bool: + if self._state.camera_open: + return False + moved = self.slam(side, now, 'giant fiber') is not None + setattr(self._refrac, side, now + self._motor_sec) + return moved + + def slam(self, side: str, now: float, cause: str): + self._hold[side] = 1.0 + if self.closed[side]: + return None + _log.warning('%s %s fired, door closed', side, cause) + self._telemetry.record_door_panic(side, cause) + self.closed[side] = True + self._closed_at[side] = now + return self._press(side, close=True) + + def release_decayed(self, now: float, frame_elapsed: float): + decay = self._hold_leak ** (frame_elapsed * self._nominal_fps) + for side in SIDES: + if not self.closed[side]: + continue + self._state.blind_until[side] = now + self._reopen_settle + self._hold[side] *= decay + if self._hold[side] >= self._release_threshold or self._state.camera_open: + continue + held = now - self._closed_at[side] + _log.info('%s escape drive decayed after %.1fs, door released', side, held) + self._telemetry.record_door_release(side, held) + self._press(side, close=False) + self.closed[side] = False + + def _press(self, side: str, close: bool): + if close: + return getattr(self._controller, f'trigger_{side}_door')() + return getattr(self._controller, f'open_{side}_door')() diff --git a/src/night/monitor.py b/src/night/monitor.py index 76ee7d3..613f24c 100644 --- a/src/night/monitor.py +++ b/src/night/monitor.py @@ -7,6 +7,7 @@ from env.vision import FNAFVision from night.motor import await_motor from night.state import SensoryState +from night.tablet.watch import IdleTablet _log = logging.getLogger(__name__) @@ -66,6 +67,17 @@ async def lower_monitor( return False +async def escape_from_tablet( + vision: FNAFVision, controller: FNAFController, lower_settle_sec: float, + settle_sec: float, confirm_sec: float, reflex, +) -> bool: + await await_motor(controller.close_camera(force=True)) + await asyncio.sleep(lower_settle_sec) + await reflex() + await await_motor(controller.centre_view()) + return await reanchor_office_reference(vision, settle_sec, confirm_sec, controller) + + def release_reason(explore, bored: bool) -> str: if not bored: return 'cap' @@ -73,12 +85,14 @@ def release_reason(explore, bored: bool) -> str: class MonitorControl: - def __init__(self, vision, controller, telemetry, state: SensoryState): + def __init__(self, vision, controller, telemetry, state: SensoryState, tablet=None): foraging = config.FORAGING_PARAMS self._vision = vision self._controller = controller self._telemetry = telemetry self._state = state + self._tablet = tablet or IdleTablet() + self._lower_settle_sec = config.TABLET_VISION.lower_settle_sec self._watch_max_sec = foraging.camera_watch_max_sec self._watch_min_sec = foraging.camera_watch_min_sec self._release_bias = foraging.camera_release_forage_bias @@ -118,6 +132,21 @@ def _raise(self, now: float, explore): self._state.camera_open = True self._opened_at = now self._cap_at = now + self._watch_max_sec + self._tablet.on_raise(now) + + def escape(self, now: float, reflex) -> bool: + if not self.is_open or self._lowering is not None: + return False + if not self._releasing: + self._telemetry.record_camera_release(now - self._opened_at, 'escape') + self._releasing = True + self._tablet.on_lower() + self._lowering = asyncio.create_task(escape_from_tablet( + self._vision, self._controller, self._lower_settle_sec, + self._settle_sec, self._confirm_sec, reflex, + )) + self._attempts = max(self._attempts, 1) + return True def _release(self, now: float, explore, forage_bias: float): watched_for = now - self._opened_at @@ -128,6 +157,7 @@ def _release(self, now: float, explore, forage_bias: float): if now < self._cap_at and not bored: return self._releasing = True + self._tablet.on_lower() self._telemetry.record_camera_release(watched_for, release_reason(explore, bored)) if now < self._retry_at: return diff --git a/src/night/tablet/__init__.py b/src/night/tablet/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/night/tablet/gaze.py b/src/night/tablet/gaze.py new file mode 100644 index 0000000..fd30c76 --- /dev/null +++ b/src/night/tablet/gaze.py @@ -0,0 +1,37 @@ +class TabletGaze: + def __init__(self, cameras: tuple, first_camera: str, raise_settle_sec: float, + switch_settle_sec: float): + self._cameras = {camera.name: camera for camera in cameras} + self._first = first_camera + self._raise_settle = raise_settle_sec + self._switch_settle = switch_settle_sec + self.camera = None + self.readable_at = 0.0 + self._pursued = False + + def spec(self, name: str | None = None): + return self._cameras.get(name or self.camera) + + def on_raised(self, now: float) -> str: + self.camera = self._first + self.readable_at = now + self._raise_settle + self._pursued = False + return self.camera + + def on_lowered(self): + self.camera = None + self._pursued = False + + def readable(self, now: float) -> bool: + return self.camera is not None and now >= self.readable_at + + def pursue(self, now: float, explore_fired: bool) -> str | None: + current = self.spec() + if current is None or current.pursue_to is None or self._pursued: + return None + if not explore_fired or not self.readable(now): + return None + self._pursued = True + self.camera = current.pursue_to + self.readable_at = now + self._switch_settle + return self.camera diff --git a/src/night/tablet/references.py b/src/night/tablet/references.py new file mode 100644 index 0000000..5c9052f --- /dev/null +++ b/src/night/tablet/references.py @@ -0,0 +1,45 @@ +import asyncio +import logging + +import config +from night.monitor import lower_monitor +from night.motor import await_motor + +_log = logging.getLogger(__name__) + + +async def capture_camera_references(feed, controller, vision, params=None) -> bool: + params = params or config.TABLET_VISION + await await_motor(controller.open_camera()) + await asyncio.sleep(params.raise_settle_sec) + if not vision.is_camera_up(): + _log.warning('the tablet did not come up for the camera references, so the fly ' + 'will raise it tonight without reading the feed') + await put_tablet_down(vision, controller) + return False + + feed.activate() + try: + recorded = [await _record(feed, controller, camera.name, params) for camera in params.cameras] + finally: + feed.deactivate() + await put_tablet_down(vision, controller) + return all(recorded) + + +async def _record(feed, controller, camera: str, params) -> bool: + await await_motor(controller.select_camera(camera)) + await asyncio.sleep(params.switch_settle_sec) + return await asyncio.get_event_loop().run_in_executor(None, feed.capture_reference, camera) + + +async def put_tablet_down(vision, controller) -> bool: + detection = config.CAMERA_DETECTION + for attempt in range(detection.lower_gesture_attempts): + if await lower_monitor( + vision, controller, detection.close_settle_sec, detection.lower_confirm_sec, + attempt, detection.lower_gesture_attempts, + ): + return True + _log.warning('the tablet stayed up after the camera references, lower it by hand') + return False diff --git a/src/night/tablet/senses.py b/src/night/tablet/senses.py new file mode 100644 index 0000000..5219a39 --- /dev/null +++ b/src/night/tablet/senses.py @@ -0,0 +1,39 @@ +import config +from night.state import SensoryState +from night.tablet.transduction import LoomChannel, camera_drive, figure_level + + +class TabletSenses: + def __init__(self, feed, params=None): + self._p = params or config.TABLET_VISION + self._feed = feed + self._cameras = {camera.name: camera for camera in self._p.cameras} + self._loom = LoomChannel( + self._p.loom_span_mse, self._p.loom_speed_span_mse_per_sec, self._p.loom_speed_decay_sec, + ) + self._watching = None + + def read(self, state: SensoryState, now: float) -> dict: + camera = state.tablet_camera if state.camera_open else None + if camera != self._watching: + self._watching = camera + self._loom.reset() + spec = self._cameras.get(camera) + if spec is None or now < state.tablet_readable_at: + return {} + contrast = self._feed.contrast(camera) + if contrast is None: + return {} + return camera_drive(spec.channel, spec.side, self._levels(spec, contrast, now)) + + def _levels(self, spec, contrast: float, now: float) -> dict: + noise = self._feed.noise(spec.name) + if spec.channel == 'figure': + return {'figure': figure_level(contrast, noise, self._p.figure_span_mse)} + size, speed = self._loom.update(contrast, noise, now) + return {'size': size, 'speed': speed} + + +class BlindSenses: + def read(self, state: SensoryState, now: float) -> dict: + return {} diff --git a/src/night/tablet/transduction.py b/src/night/tablet/transduction.py new file mode 100644 index 0000000..4c973ac --- /dev/null +++ b/src/night/tablet/transduction.py @@ -0,0 +1,44 @@ +import math + + +def clamp_unit(value: float) -> float: + return min(1.0, max(0.0, value)) + + +def figure_level(contrast: float, noise: float, span: float) -> float: + return clamp_unit((contrast - noise) / span) + + +class LoomChannel: + def __init__(self, size_span: float, speed_span_per_sec: float, speed_decay_sec: float): + self._size_span = size_span + self._speed_span = speed_span_per_sec + self._decay_sec = speed_decay_sec + self._previous = None + self._previous_at = 0.0 + self.speed = 0.0 + + def reset(self): + self._previous = None + self.speed = 0.0 + + def update(self, contrast: float, noise: float, now: float) -> tuple: + size = clamp_unit((contrast - noise) / self._size_span) + if self._previous is None: + self._previous, self._previous_at = contrast, now + return size, 0.0 + elapsed = max(now - self._previous_at, 1e-3) + growth = max(0.0, contrast - self._previous) / elapsed + held = self.speed * math.exp(-elapsed / self._decay_sec) + self.speed = max(held, clamp_unit(growth / self._speed_span)) + self._previous, self._previous_at = contrast, now + return size, self.speed + + +def camera_drive(channel: str, side: str, levels: dict) -> dict: + if channel == 'figure': + return {f'figure_{side}': levels.get('figure', 0.0)} + return { + f'loom_size_{side}': levels.get('size', 0.0), + f'loom_speed_{side}': levels.get('speed', 0.0), + } diff --git a/src/night/tablet/watch.py b/src/night/tablet/watch.py new file mode 100644 index 0000000..6349999 --- /dev/null +++ b/src/night/tablet/watch.py @@ -0,0 +1,72 @@ +import logging + +import config +from night.state import SensoryState +from night.tablet.gaze import TabletGaze + +_log = logging.getLogger(__name__) + + +def build_gaze(params=None) -> TabletGaze: + params = params or config.TABLET_VISION + return TabletGaze( + params.cameras, params.first_camera, params.raise_settle_sec, params.switch_settle_sec, + ) + + +class TabletWatch: + def __init__(self, gaze: TabletGaze, feed, controller, telemetry, state: SensoryState, + map_ready_sec: float | None = None): + self._gaze = gaze + self._map_ready_sec = ( + config.TABLET_VISION.map_ready_sec if map_ready_sec is None else map_ready_sec + ) + self._feed = feed + self._controller = controller + self._telemetry = telemetry + self._state = state + + @property + def camera(self) -> str | None: + return self._gaze.camera + + def on_raise(self, now: float): + self._feed.activate() + self._show(self._gaze.on_raised(now), 'raise', self._map_ready_sec) + + def on_lower(self): + self._gaze.on_lowered() + self._feed.deactivate() + self._state.tablet_camera = None + self._state.tablet_drive = {} + + def update(self, now: float, explore_fired: bool, escape_fired: dict) -> str | None: + if self._gaze.camera is None: + return None + switched = self._gaze.pursue(now, explore_fired) + if switched is not None: + _log.info('DNp09 fired while watching, gaze follows to camera %s', switched) + self._show(switched, 'pursuit', 0.0) + for side in ('left', 'right'): + if escape_fired.get(side): + return side + return None + + def _show(self, camera: str, cause: str, settle_sec: float): + self._controller.select_camera(camera, settle_sec) + self._state.tablet_camera = camera + self._state.tablet_readable_at = self._gaze.readable_at + self._telemetry.record_camera_view(camera, cause) + + +class IdleTablet: + camera = None + + def on_raise(self, now: float): + return None + + def on_lower(self): + return None + + def update(self, now: float, explore_fired: bool, escape_fired: dict) -> str | None: + return None diff --git a/src/night/tasks.py b/src/night/tasks.py index 52a01bd..82a05c3 100644 --- a/src/night/tasks.py +++ b/src/night/tasks.py @@ -10,7 +10,10 @@ from night.engine import ConnectomeEngine from night.monitor import MonitorControl from night.motor import await_motor -from night.state import MotorRefrac, SaccadeRequest, SensoryState +from night.doors import DoorControl +from night.state import SaccadeRequest, SensoryState +from night.tablet.senses import BlindSenses +from night.tablet.watch import IdleTablet from recorder import SessionRecorder from search_drive import ExploreDrive, SearchDrive from telemetry import ConnectomeTelemetry @@ -19,9 +22,11 @@ async def vision_task( - vision: FNAFVision, controller: FNAFController, state: SensoryState, shutdown: asyncio.Event + vision: FNAFVision, controller: FNAFController, state: SensoryState, shutdown: asyncio.Event, + senses=None, ): delay = config.VISION_DYNAMICS.capture_delay_sec + senses = senses or BlindSenses() while not shutdown.is_set(): state.office_centred = controller.facing() == 'centre' cam_up = state.office_centred and vision.is_camera_up() @@ -31,6 +36,7 @@ async def vision_task( state.left_rate = vision.get_left_sensory_rate() if looking_left else 0.0 state.right_rate = vision.get_right_sensory_rate() if looking_right else 0.0 state.cam_inhib = config.SIMULATION_PARAMS.base_sensory_rate_hz if cam_up else 0.0 + state.tablet_drive = senses.read(state, now) await asyncio.sleep(delay) @@ -74,9 +80,10 @@ async def saccade_task( saccade: SaccadeRequest, view_ready: asyncio.Event, begin_night=None, + tablet_feed=None, ): await view_ready.wait() - await calibrate(vision, controller, begin_night) + await calibrate(vision, controller, begin_night, tablet_feed) calibration_done.set() while not shutdown.is_set(): @@ -126,27 +133,17 @@ async def engine_task( highlights: dict, saccade: SaccadeRequest, trace: SessionRecorder | None = None, + tablet=None, ): await calibration_done.wait() trace = trace or SessionRecorder(enabled=False) + tablet = tablet or IdleTablet() search = SearchDrive(engine.sensory_span) explore = ExploreDrive() - monitor = MonitorControl(vision, controller, telemetry, state) - refrac = MotorRefrac() - motor_dur = config.FORAGING_PARAMS.motor_refractory_sec - cam_stuck_warn_sec = config.CAMERA_DETECTION.stuck_warn_sec - inhib_since = 0.0 - inhib_warned_at = 0.0 - - doors = (('left', controller.open_left_door), ('right', controller.open_right_door)) - door_closed = {'left': False, 'right': False} - door_closed_at = {'left': 0.0, 'right': 0.0} - door_hold = {'left': 0.0, 'right': 0.0} - hold_leak = config.DOOR_DYNAMICS.hold_leak_per_frame - hold_release = config.DOOR_DYNAMICS.release_threshold - reopen_settle = config.DOOR_DYNAMICS.reopen_settle_sec - nominal_fps = config.SIMULATION_PARAMS.target_fps + monitor = MonitorControl(vision, controller, telemetry, state, tablet) + doors = DoorControl(controller, telemetry, state) + watchdog = InhibitionWatchdog(vision) last_frame_at = time.time() while not shutdown.is_set(): @@ -158,125 +155,137 @@ async def engine_task( inputs = (state.left_rate > 0.0, state.right_rate > 0.0) loop = asyncio.get_event_loop() spikes = await loop.run_in_executor(None, engine.step, state, now) + read_spikes(engine, spikes, state) - state.l_spike = bool(spikes[0, engine.l_motor_idx].any()) - state.r_spike = bool(spikes[0, engine.r_motor_idx].any()) - state.l_sensory_count = int(spikes[0, engine.l_sensory_idx].sum().item()) - state.r_sensory_count = int(spikes[0, engine.r_sensory_idx].sum().item()) - - explore.update( - engine.explore_membrane, - bool(spikes[0, engine.explore_idx].any()), - frame_elapsed, - ) + explore.update(engine.explore_membrane, state.explore_spike, frame_elapsed) look_pending = saccade.side is not None or saccade.busy telemetry.record_frame(state.cam_inhib > 0, look_pending) monitor.update(now, explore, state.forage_bias, look_pending) - - can_look = not look_pending and not monitor.is_open and now >= saccade.ready_at - wants = search.update( - engine.eye_membrane_diff, - state.l_sensory_count, - state.r_sensory_count, - frame_elapsed, - now, - can_look, - ) - state.forage_bias = search.tension - if wants is not None: - saccade.reason = search.reason - saccade.drive = search.last_drive - saccade.side = wants + watch_the_tablet(tablet, monitor, doors, telemetry, state, now) + request_saccade(search, saccade, monitor, engine, state, frame_elapsed, now, look_pending) feed.publish(spikes[0].nonzero().flatten().cpu().numpy()) - highlights['gf_l'] = state.l_spike - highlights['gf_r'] = state.r_spike - highlights['camera'] = monitor.is_open - highlights['in_l'], highlights['in_r'] = inputs - highlights['look_l'] = state.check_left - highlights['look_r'] = state.check_right - highlights['mse_l'] = vision.last_mse('left') if state.check_left else None - highlights['mse_r'] = vision.last_mse('right') if state.check_right else None - highlights['mse_th'] = vision.mse_threshold - highlights['eye_l'] = state.l_sensory_count - highlights['eye_r'] = state.r_sensory_count - highlights['door_l'] = door_closed['left'] - highlights['door_r'] = door_closed['right'] - if state.check_left: - highlights['gaze'] = 'LEFT' - elif state.check_right: - highlights['gaze'] = 'RIGHT' - else: - highlights['gaze'] = '--' - + fill_highlights(highlights, state, vision, doors, monitor, inputs) trace.frame( now - telemetry.start_time, state, engine, state.l_spike, state.r_spike, vision.peak_mse('left'), vision.peak_mse('right'), monitor.is_open, ) - - if state.cam_inhib > 0 and not monitor.is_open: - if inhib_since == 0.0: - inhib_since = now - elif now - inhib_since > cam_stuck_warn_sec and now - inhib_warned_at > 30.0: - _log.warning( - 'the office reference has disagreed with the screen for %.0fs at %.0f mse; ' - 'the inhibitors are being driven with the tablet down, which silences ' - 'the giant fiber and DNp09. Recapture it with the office on screen.', - now - inhib_since, vision.camera_mse(), - ) - inhib_warned_at = now - else: - inhib_since = 0.0 - - decay = hold_leak ** (frame_elapsed * nominal_fps) - - for side, open_door in doors: - if not door_closed[side]: - continue - state.blind_until[side] = now + reopen_settle - door_hold[side] *= decay - if door_hold[side] >= hold_release or state.camera_open: - continue - held = now - door_closed_at[side] - _log.info('%s escape drive decayed after %.1fs, door released', side, held) - telemetry.record_door_release(side, held) - open_door() - door_closed[side] = False - - if spikes[0, engine.l_motor_idx].any() and now > refrac.left: - moved = False - if not state.camera_open: - door_hold['left'] = 1.0 - if not door_closed['left']: - _log.warning('left giant fiber fired') - telemetry.record_door_panic('left') - controller.trigger_left_door() - door_closed['left'] = True - door_closed_at['left'] = now - moved = True - refrac.left = now + motor_dur - feed.trace_escape('left', engine.l_motor_idx[0], engine.l_sensory_idx, moved) - state.left_rate = 0.0 - - if spikes[0, engine.r_motor_idx].any() and now > refrac.right: - moved = False - if not state.camera_open: - door_hold['right'] = 1.0 - if not door_closed['right']: - _log.warning('right giant fiber fired') - telemetry.record_door_panic('right') - controller.trigger_right_door() - door_closed['right'] = True - door_closed_at['right'] = now - moved = True - refrac.right = now + motor_dur - feed.trace_escape('right', engine.r_motor_idx[0], engine.r_sensory_idx, moved) - state.right_rate = 0.0 - - elapsed = time.perf_counter() - t_start - remaining = engine.frame_dt - elapsed - if remaining > 0: - await asyncio.sleep(remaining) - else: - await asyncio.sleep(0) + watchdog.check(state, monitor, now) + doors.release_decayed(now, frame_elapsed) + answer_giant_fibers(engine, doors, feed, state, now) + + remaining = engine.frame_dt - (time.perf_counter() - t_start) + await asyncio.sleep(max(remaining, 0.0)) + + +def read_spikes(engine, spikes, state: SensoryState): + frame = spikes[0] + state.l_spike = bool(frame[engine.l_motor_idx].any()) + state.r_spike = bool(frame[engine.r_motor_idx].any()) + state.l_escape = bool(frame[engine.l_escape_idx].any()) + state.r_escape = bool(frame[engine.r_escape_idx].any()) + state.explore_spike = bool(frame[engine.explore_idx].any()) + state.l_sensory_count = int(frame[engine.l_sensory_idx].sum().item()) + state.r_sensory_count = int(frame[engine.r_sensory_idx].sum().item()) + + +def watch_the_tablet(tablet, monitor: MonitorControl, doors: DoorControl, telemetry, + state: SensoryState, now: float): + if not monitor.is_open: + return + camera = tablet.camera + side = tablet.update(now, state.explore_spike, {'left': state.l_escape, 'right': state.r_escape}) + if side is None: + return + + async def close_the_door(): + pressed = doors.slam(side, now, 'DNp04 looming escape') + if pressed is not None: + await await_motor(pressed) + + if monitor.escape(now, close_the_door): + _log.warning('%s DNp04 fired while watching camera %s, dropping the tablet for the door', + side, camera) + telemetry.record_tablet_escape(side, camera) + + +def request_saccade(search: SearchDrive, saccade: SaccadeRequest, monitor: MonitorControl, + engine, state: SensoryState, frame_elapsed: float, now: float, + look_pending: bool): + can_look = not look_pending and not monitor.is_open and now >= saccade.ready_at + wants = search.update( + engine.eye_membrane_diff, state.l_sensory_count, state.r_sensory_count, + frame_elapsed, now, can_look, + ) + state.forage_bias = search.tension + if wants is None: + return + saccade.reason = search.reason + saccade.drive = search.last_drive + saccade.side = wants + + +def answer_giant_fibers(engine, doors: DoorControl, feed: SpikeFeed, state: SensoryState, now: float): + for side, fired, motor_idx, sensory_idx in ( + ('left', state.l_spike, engine.l_motor_idx, engine.l_sensory_idx), + ('right', state.r_spike, engine.r_motor_idx, engine.r_sensory_idx), + ): + if not fired or doors.refractory(side, now): + continue + moved = doors.on_giant_fiber(side, now) + feed.trace_escape(side, motor_idx[0], sensory_idx, moved) + setattr(state, f'{side}_rate', 0.0) + + +def _strongest(drive: dict, prefix: str) -> float: + return max((level for name, level in drive.items() if name.startswith(prefix)), default=0.0) + + +def fill_highlights(highlights: dict, state: SensoryState, vision: FNAFVision, + doors: DoorControl, monitor: MonitorControl, inputs: tuple): + highlights['gf_l'] = state.l_spike + highlights['gf_r'] = state.r_spike + highlights['camera'] = monitor.is_open + highlights['in_l'], highlights['in_r'] = inputs + highlights['look_l'] = state.check_left + highlights['look_r'] = state.check_right + highlights['mse_l'] = vision.last_mse('left') if state.check_left else None + highlights['mse_r'] = vision.last_mse('right') if state.check_right else None + highlights['mse_th'] = vision.mse_threshold + highlights['eye_l'] = state.l_sensory_count + highlights['eye_r'] = state.r_sensory_count + highlights['door_l'] = doors.closed['left'] + highlights['door_r'] = doors.closed['right'] + highlights['cam'] = state.tablet_camera + highlights['figure'] = _strongest(state.tablet_drive, 'figure') + highlights['loom'] = _strongest(state.tablet_drive, 'loom') + highlights['dnp09'] = state.explore_spike + highlights['dnp04_l'] = state.l_escape + highlights['dnp04_r'] = state.r_escape + highlights['gaze'] = 'LEFT' if state.check_left else 'RIGHT' if state.check_right else '--' + + +class InhibitionWatchdog: + def __init__(self, vision: FNAFVision): + self._vision = vision + self._warn_after = config.CAMERA_DETECTION.stuck_warn_sec + self._since = 0.0 + self._warned_at = 0.0 + + def check(self, state: SensoryState, monitor: MonitorControl, now: float): + if state.cam_inhib <= 0 or monitor.is_open: + self._since = 0.0 + return + if self._since == 0.0: + self._since = now + return + if now - self._since <= self._warn_after or now - self._warned_at <= 30.0: + return + _log.warning( + 'the office reference has disagreed with the screen for %.0fs at %.0f mse; ' + 'the inhibitors are being driven with the tablet down, which silences ' + 'the giant fiber and DNp09. Recapture it with the office on screen.', + now - self._since, self._vision.camera_mse(), + ) + self._warned_at = now diff --git a/src/telemetry.py b/src/telemetry.py index 8903575..5a3f522 100644 --- a/src/telemetry.py +++ b/src/telemetry.py @@ -23,7 +23,10 @@ def __init__(self): self.saccade_reasons = {'drift': 0, 'evidence': 0, 'guard': 0} self.camera_causes = {'spike': 0, 'drive': 0} self.camera_watches = [] - self.camera_release_reasons = {'drive': 0, 'search': 0, 'cap': 0} + self.camera_release_reasons = {'drive': 0, 'search': 0, 'cap': 0, 'escape': 0} + self.camera_views = {} + self.camera_view_causes = {'raise': 0, 'pursuit': 0} + self.tablet_escapes = {} self.look_contrast = {'left': [], 'right': []} self.look_frames = [] self.look_driven = [] @@ -34,10 +37,26 @@ def __init__(self): self.saccade_gaps = [] self._last_saccade_at = None - def record_door_panic(self, side: str): + def record_door_panic(self, side: str, cause: str = 'giant fiber'): t = time.time() - self.start_time self.stats[f'{side}_door_panics'] += 1 - self.events.append(f"[{t:>6.1f}s] Visual threat detected. Giant fiber fired. {side.capitalize()} door slammed.") + self.events.append(f"[{t:>6.1f}s] Visual threat detected. {cause.capitalize()} fired. {side.capitalize()} door slammed.") + + def record_camera_view(self, camera: str, cause: str): + t = time.time() - self.start_time + self.camera_views[camera] = self.camera_views.get(camera, 0) + 1 + if cause in self.camera_view_causes: + self.camera_view_causes[cause] += 1 + why = 'the tablet came up on it' if cause == 'raise' else 'DNp09 fired and the gaze followed' + self.events.append(f"[{t:>6.1f}s] Watching CAM {camera}, {why}.") + + def record_tablet_escape(self, side: str, camera: str): + t = time.time() - self.start_time + key = f'{camera} {side}' + self.tablet_escapes[key] = self.tablet_escapes.get(key, 0) + 1 + self.events.append( + f"[{t:>6.1f}s] DNp04 {side} fired on a looming shape in CAM {camera}. Tablet dropped for the door." + ) def record_door_release(self, side: str, held_sec: float): t = time.time() - self.start_time @@ -57,6 +76,7 @@ def record_camera_pull(self, commanded: bool, drive: float): 'drive': 'the exploratory drive faded', 'search': 'the fly wanted to check a hallway', 'cap': 'the power cap expired', + 'escape': 'something loomed on the camera', } def record_camera_release(self, watched_sec: float, reason: str): @@ -108,6 +128,18 @@ def record_light_saccade(self, side: str, drive: float, reason: str): f"[{t:>6.1f}s] {cause} drove a {side} light check (search drive: {drive:+.2f})." ) + def _write_tablet(self, f): + f.write("--- What The Tablet Showed ---\n") + if not self.camera_views: + f.write(f"{'Cameras watched':<38}: none\n\n") + return + views = ', '.join(f'{camera} x{count}' for camera, count in sorted(self.camera_views.items())) + f.write(f"{'Cameras watched':<38}: {views}\n") + causes = self.camera_view_causes + f.write(f"{'Chosen by':<38}: {causes['raise']} on raising, {causes['pursuit']} DNp09 pursuit\n") + escapes = ', '.join(f'{key} x{count}' for key, count in sorted(self.tablet_escapes.items())) or 'none' + f.write(f"{'DNp04 escapes from the tablet':<38}: {escapes}\n\n") + def dump_report(self): duration = time.time() - self.start_time os.makedirs("logs", exist_ok=True) @@ -135,7 +167,7 @@ def dump_report(self): f.write(f"Mean Monitor Watch : {mean:.1f}s, {sum(self.camera_watches) / max(duration, 1.0) * 100:.0f}% of the night blind\n") f.write(f"Monitor Raised By : {self.camera_causes['spike']} DNp09 spikes, {self.camera_causes['drive']} accumulated drive\n") by = self.camera_release_reasons - f.write(f"Monitor Lowered By : {by['drive']} drive faded, {by['search']} hallway won, {by['cap']} power cap\n") + f.write(f"Monitor Lowered By : {by['drive']} drive faded, {by['search']} hallway won, {by['cap']} power cap, {by['escape']} looming escape\n") f.write(f"Left Light Checks : {self.stats['left_light_saccades']}\n") f.write(f"Right Light Checks : {self.stats['right_light_saccades']}\n\n") @@ -155,6 +187,8 @@ def dump_report(self): f.write("\n") + self._write_tablet(f) + f.write("--- Where The Night Went ---\n") if self.engine_frames: rate = self.engine_frames / max(duration, 1.0) diff --git a/tests/test_search_loop_wiring.py b/tests/test_search_loop_wiring.py index 735e3ed..3c823f6 100644 --- a/tests/test_search_loop_wiring.py +++ b/tests/test_search_loop_wiring.py @@ -47,6 +47,8 @@ def __init__(self): self.explore_idx = [2] self.l_sensory_idx = [3] self.r_sensory_idx = [4] + self.l_escape_idx = [5] + self.r_escape_idx = [6] self.frames = 0 self.driven_frames = {'left': 0, 'right': 0} self._t0 = None diff --git a/tests/test_tablet_escape.py b/tests/test_tablet_escape.py new file mode 100644 index 0000000..9db506f --- /dev/null +++ b/tests/test_tablet_escape.py @@ -0,0 +1,165 @@ +import asyncio +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) + +from night import monitor, tasks +from night.doors import DoorControl +from night.state import SensoryState +from telemetry import ConnectomeTelemetry + + +class _Done: + def wait(self, timeout=None): + return True + + +class RecordingController: + def __init__(self): + self.calls = [] + + def facing(self): + return 'centre' + + def __getattr__(self, name): + if name.startswith('_'): + raise AttributeError(name) + + def record(*args, **kwargs): + self.calls.append(name) + return _Done() + return record + + +class OfficeVision: + def clear_buffers(self): + pass + + def is_camera_up(self): + return False + + def capture_camera_closed_reference(self): + pass + + def camera_mse(self): + return 0.0 + + +class StubTablet: + camera = '2A' + + def __init__(self, escape_side): + self.escape_side = escape_side + self.lowered = 0 + + def on_raise(self, now): + pass + + def on_lower(self): + self.lowered += 1 + + def update(self, now, explore_fired, escape_fired): + return self.escape_side + + +class StubExplore: + wants_monitor = True + spent = False + commanded = False + drive = 1.2 + + +def _raised(escape_side='left'): + state = SensoryState() + controller = RecordingController() + telemetry = ConnectomeTelemetry() + tablet = StubTablet(escape_side) + control = monitor.MonitorControl(OfficeVision(), controller, telemetry, state, tablet) + control.update(1.0, StubExplore(), 0.0, look_pending=False) + doors = DoorControl(controller, telemetry, state) + return control, doors, controller, telemetry, state, tablet + + +async def _escape_and_settle(escape_side='left'): + control, doors, controller, telemetry, state, tablet = _raised(escape_side) + tasks.watch_the_tablet(tablet, control, doors, telemetry, state, now=2.0) + for _ in range(300): + control.update(3.0, StubExplore(), 0.0, look_pending=False) + if not control.is_open: + break + await asyncio.sleep(0.02) + return control, doors, controller, telemetry, state, tablet + + +def _run(escape_side='left'): + return asyncio.run(_escape_and_settle(escape_side)) + + +def test_the_tablet_goes_down_before_the_door_and_the_view_comes_back_after(): + control, doors, controller, telemetry, state, tablet = _run('left') + order = [call for call in controller.calls if call != 'select_camera'] + assert order[:4] == ['open_camera', 'close_camera', 'trigger_left_door', 'centre_view'], order + assert not control.is_open + assert doors.closed['left'] and not doors.closed['right'] + assert state.camera_open is False + + +def test_the_escape_is_reported_as_its_own_release_and_door_cause(): + control, doors, controller, telemetry, state, tablet = _run('right') + assert telemetry.camera_release_reasons['escape'] == 1 + assert telemetry.stats['right_door_panics'] == 1 + assert telemetry.tablet_escapes == {'2A right': 1} + assert tablet.lowered >= 1 + + +def test_no_escape_is_started_without_a_dnp04_spike(): + control, doors, controller, telemetry, state, tablet = _raised(escape_side=None) + tasks.watch_the_tablet(tablet, control, doors, telemetry, state, now=2.0) + assert control.is_open + assert 'close_camera' not in controller.calls + assert not any(doors.closed.values()) + + +def test_a_second_escape_waits_for_the_first_lowering(): + async def twice(): + control, doors, controller, telemetry, state, tablet = _raised('left') + first = control.escape(2.0, _noop) + second = control.escape(2.1, _noop) + await asyncio.sleep(0) + return first, second + assert asyncio.run(twice()) == (True, False) + + +async def _noop(): + return None + + +def test_the_giant_fiber_leaves_the_door_alone_while_the_tablet_is_up(): + state = SensoryState(camera_open=True) + controller = RecordingController() + doors = DoorControl(controller, ConnectomeTelemetry(), state) + assert doors.on_giant_fiber('left', now=1.0) is False + assert controller.calls == [] + assert not doors.refractory('left', now=1.0) + + +def test_a_door_is_slammed_once_and_reopened_when_the_hold_decays(): + state = SensoryState() + controller = RecordingController() + doors = DoorControl(controller, ConnectomeTelemetry(), state) + assert doors.on_giant_fiber('left', now=1.0) is True + assert doors.on_giant_fiber('left', now=5.0) is False + assert controller.calls == ['trigger_left_door'] + + for frame in range(200): + doors.release_decayed(now=6.0 + frame * 0.1, frame_elapsed=0.1) + assert controller.calls == ['trigger_left_door', 'open_left_door'] + assert not doors.closed['left'] + + +if __name__ == '__main__': + for name, fn in sorted(globals().items()): + if name.startswith('test_'): + fn() + print('ok') diff --git a/tests/test_tablet_gaze.py b/tests/test_tablet_gaze.py new file mode 100644 index 0000000..3c55c18 --- /dev/null +++ b/tests/test_tablet_gaze.py @@ -0,0 +1,125 @@ +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) + +import config +from night.state import SensoryState +from night.tablet.gaze import TabletGaze +from night.tablet.watch import TabletWatch +from telemetry import ConnectomeTelemetry + +RAISE = 1.2 +SWITCH = 0.5 + + +def _gaze() -> TabletGaze: + return TabletGaze(config.TABLET_VISION.cameras, '1C', RAISE, SWITCH) + + +class _Done: + def wait(self, timeout=None): + return True + + +class RecordingController: + def __init__(self): + self.selected = [] + + def select_camera(self, camera, settle_sec=0.0): + self.selected.append((camera, settle_sec)) + return _Done() + + +class RecordingFeed: + def __init__(self): + self.active = False + + def activate(self): + self.active = True + + def deactivate(self): + self.active = False + + +def test_raising_the_tablet_always_opens_on_the_cove(): + gaze = _gaze() + assert gaze.on_raised(now=10.0) == '1C' + assert gaze.readable_at == 10.0 + RAISE + assert not gaze.readable(10.0) + assert gaze.readable(10.0 + RAISE) + + +PURSUIT_CASES = ( + ('no spike', 20.0, False, None), + ('spike during the raise static', 10.1, True, None), + ('spike once the cove is readable', 20.0, True, '2A'), +) + + +def test_pursuit_needs_a_dnp09_spike_on_a_readable_cove(): + for label, now, fired, expected in PURSUIT_CASES: + gaze = _gaze() + gaze.on_raised(now=10.0) + assert gaze.pursue(now, fired) == expected, label + + +def test_pursuit_happens_once_per_raise_and_never_from_the_hall(): + gaze = _gaze() + gaze.on_raised(now=0.0) + assert gaze.pursue(5.0, True) == '2A' + assert gaze.camera == '2A' + assert gaze.pursue(10.0, True) is None + + gaze.on_lowered() + gaze.on_raised(now=20.0) + assert gaze.pursue(25.0, True) == '2A' + + +def test_the_watch_taps_the_map_and_tells_the_senses_where_it_looks(): + state = SensoryState(camera_open=True) + controller, feed, telemetry = RecordingController(), RecordingFeed(), ConnectomeTelemetry() + watch = TabletWatch(_gaze(), feed, controller, telemetry, state) + + watch.on_raise(now=0.0) + assert feed.active + assert state.tablet_camera == '1C' + assert state.tablet_readable_at == RAISE + + watch.update(5.0, explore_fired=True, escape_fired={}) + assert controller.selected == [('1C', config.TABLET_VISION.map_ready_sec), ('2A', 0.0)] + assert state.tablet_camera == '2A' + assert telemetry.camera_view_causes == {'raise': 1, 'pursuit': 1} + + watch.on_lower() + assert not feed.active + assert state.tablet_camera is None + assert state.tablet_drive == {} + + +ESCAPE_CASES = ( + ({'left': True, 'right': False}, 'left'), + ({'left': False, 'right': True}, 'right'), + ({'left': False, 'right': False}, None), +) + + +def test_a_dnp04_spike_while_watching_names_the_door(): + for fired, expected in ESCAPE_CASES: + watch = TabletWatch(_gaze(), RecordingFeed(), RecordingController(), ConnectomeTelemetry(), + SensoryState(camera_open=True)) + watch.on_raise(now=0.0) + assert watch.update(0.1, explore_fired=False, escape_fired=fired) == expected, fired + + +def test_nothing_is_read_from_the_watch_while_the_tablet_is_down(): + watch = TabletWatch(_gaze(), RecordingFeed(), RecordingController(), ConnectomeTelemetry(), + SensoryState()) + assert watch.update(1.0, explore_fired=True, escape_fired={'left': True}) is None + + +if __name__ == '__main__': + for name, fn in sorted(globals().items()): + if name.startswith('test_'): + fn() + print('ok') diff --git a/tests/test_tablet_senses.py b/tests/test_tablet_senses.py new file mode 100644 index 0000000..44a74dc --- /dev/null +++ b/tests/test_tablet_senses.py @@ -0,0 +1,71 @@ +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) + +from night.state import SensoryState +from night.tablet.senses import TabletSenses + + +class ScriptedFeed: + def __init__(self, contrasts: dict, noise: float = 100.0): + self.contrasts = contrasts + self._noise = noise + + def contrast(self, camera): + return self.contrasts.get(camera) + + def noise(self, camera): + return self._noise + + +def _watching(camera, readable_at=0.0, camera_open=True) -> SensoryState: + return SensoryState(camera_open=camera_open, tablet_camera=camera, tablet_readable_at=readable_at) + + +SILENT_CASES = ( + ('tablet down', _watching('1C', camera_open=False)), + ('no camera chosen', _watching(None)), + ('camera still in its switching static', _watching('1C', readable_at=99.0)), + ('a camera with no reference', _watching('2A')), +) + + +def test_nothing_reaches_the_brain_unless_a_known_camera_is_readable(): + senses = TabletSenses(ScriptedFeed({'1C': 5000.0})) + for label, state in SILENT_CASES: + assert senses.read(state, now=1.0) == {}, label + + +def test_the_cove_drives_the_figure_population_on_its_side(): + senses = TabletSenses(ScriptedFeed({'1C': 600.0})) + drive = senses.read(_watching('1C'), now=1.0) + assert drive == {'figure_left': 0.5} + + +def test_the_west_hall_drives_the_looming_populations(): + feed = ScriptedFeed({'2A': 100.0}) + senses = TabletSenses(feed) + state = _watching('2A') + senses.read(state, now=1.0) + feed.contrasts['2A'] = 1100.0 + drive = senses.read(state, now=1.1) + assert set(drive) == {'loom_size_left', 'loom_speed_left'} + assert drive['loom_size_left'] == 1.0 + assert drive['loom_speed_left'] > 0.9 + + +def test_switching_cameras_does_not_count_the_cut_as_motion(): + feed = ScriptedFeed({'1C': 100.0, '2A': 5000.0}) + senses = TabletSenses(feed) + senses.read(_watching('2A'), now=0.0) + senses.read(_watching('1C'), now=0.5) + drive = senses.read(_watching('2A'), now=1.0) + assert drive['loom_speed_left'] == 0.0 + + +if __name__ == '__main__': + for name, fn in sorted(globals().items()): + if name.startswith('test_'): + fn() + print('ok') diff --git a/tests/test_tablet_transduction.py b/tests/test_tablet_transduction.py new file mode 100644 index 0000000..b3686ed --- /dev/null +++ b/tests/test_tablet_transduction.py @@ -0,0 +1,108 @@ +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) + +import numpy as np + +from env.tablet_feed import bank_distance, bank_noise, prepare, spread_sample +from night.tablet.transduction import LoomChannel, camera_drive, figure_level + +FIGURE_CASES = ( + (0.0, 100.0, 1000.0, 0.0), + (100.0, 100.0, 1000.0, 0.0), + (600.0, 100.0, 1000.0, 0.5), + (5000.0, 100.0, 1000.0, 1.0), + (50.0, 100.0, 1000.0, 0.0), +) + + +def test_figure_level_is_the_contrast_above_noise_over_the_span(): + for contrast, noise, span, expected in FIGURE_CASES: + assert abs(figure_level(contrast, noise, span) - expected) < 1e-9, (contrast, noise) + + +def _loom() -> LoomChannel: + return LoomChannel(size_span=1000.0, speed_span_per_sec=3000.0, speed_decay_sec=0.3) + + +def test_a_growing_shape_drives_speed_and_a_standing_one_only_size(): + loom = _loom() + loom.update(0.0, 0.0, now=0.0) + size, speed = loom.update(600.0, 0.0, now=0.1) + assert size == 0.6 + assert speed > 0.9 + + for step in range(2, 20): + size, speed = loom.update(600.0, 0.0, now=step * 0.1) + assert size == 0.6 + assert speed < 0.01 + + +def test_speed_is_held_briefly_so_a_slow_engine_frame_can_still_read_it(): + loom = _loom() + loom.update(0.0, 0.0, now=0.0) + _, peak = loom.update(900.0, 0.0, now=0.05) + _, after = loom.update(900.0, 0.0, now=0.10) + assert 0.0 < after < peak + + +def test_a_shape_that_shrinks_is_not_looming(): + loom = _loom() + loom.update(900.0, 0.0, now=0.0) + _, speed = loom.update(100.0, 0.0, now=0.1) + assert speed == 0.0 + + +def test_reset_forgets_the_previous_camera(): + loom = _loom() + loom.update(0.0, 0.0, now=0.0) + loom.update(900.0, 0.0, now=0.1) + loom.reset() + _, speed = loom.update(900.0, 0.0, now=0.2) + assert speed == 0.0 + + +DRIVE_CASES = ( + ('figure', 'left', {'figure': 0.4}, {'figure_left': 0.4}), + ('loom', 'left', {'size': 0.2, 'speed': 0.7}, {'loom_size_left': 0.2, 'loom_speed_left': 0.7}), + ('loom', 'right', {}, {'loom_size_right': 0.0, 'loom_speed_right': 0.0}), +) + + +def test_camera_drive_names_the_population_on_the_camera_side(): + for channel, side, levels, expected in DRIVE_CASES: + assert camera_drive(channel, side, levels) == expected, (channel, side) + + +def test_bank_distance_takes_the_nearest_reference_so_panning_is_not_a_threat(): + left = np.zeros((8, 8), np.float32) + right = np.full((8, 8), 50.0, np.float32) + assert bank_distance(right, [left, right]) == 0.0 + assert bank_distance(np.full((8, 8), 45.0, np.float32), [left, right]) == 25.0 + assert bank_distance(left, []) == 0.0 + + +def test_bank_noise_is_the_worst_leave_one_out_distance(): + frames = [np.full((4, 4), value, np.float32) for value in (0.0, 1.0, 3.0)] + assert bank_noise(frames) == 4.0 + assert bank_noise(frames[:1]) == 0.0 + + +def test_spread_sample_keeps_the_whole_sweep(): + assert spread_sample(list(range(10)), 20) == list(range(10)) + assert spread_sample(list(range(100)), 4) == [0, 25, 50, 75] + + +def test_prepare_shrinks_colour_frames_to_a_square_grey_patch(): + frame = np.random.default_rng(1).integers(0, 255, (120, 120, 4), dtype=np.uint8) + patch = prepare(frame, 32) + assert patch.shape == (32, 32) + assert patch.dtype == np.float32 + + +if __name__ == '__main__': + for name, fn in sorted(globals().items()): + if name.startswith('test_'): + fn() + print('ok') From 0ff593ea46f3ac71de1828a7acfcb25a23d98ebd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Artur=20Mendon=C3=A7a=20Arruda?= Date: Thu, 1 Oct 2026 01:50:58 -0300 Subject: [PATCH 04/14] feat(sim): record the tablet drive in the trace and replay it --- src/recorder.py | 7 ++++++- src/replay.py | 15 +++++++++++++++ tests/test_trace_roundtrip.py | 25 ++++++++++++++++++++----- 3 files changed, 41 insertions(+), 6 deletions(-) diff --git a/src/recorder.py b/src/recorder.py index 63063c0..de70819 100644 --- a/src/recorder.py +++ b/src/recorder.py @@ -8,7 +8,7 @@ _log = logging.getLogger(__name__) -TRACE_VERSION = 1 +TRACE_VERSION = 2 def default_path() -> str: @@ -73,6 +73,11 @@ def frame(self, elapsed: float, state, engine, spikes_left: bool, spikes_right: 'camera_open': bool(camera_open), 'gf_left': bool(spikes_left), 'gf_right': bool(spikes_right), + 'escape_left': bool(state.l_escape), + 'escape_right': bool(state.r_escape), + 'explore_spike': bool(state.explore_spike), + 'tablet_camera': state.tablet_camera, + 'tablet_drive': {name: round(float(level), 4) for name, level in state.tablet_drive.items()}, 'eye_membrane_diff': round(float(engine.eye_membrane_diff), 4), 'explore_membrane': round(float(engine.explore_membrane), 4), }) diff --git a/src/replay.py b/src/replay.py index 9291eab..72474be 100644 --- a/src/replay.py +++ b/src/replay.py @@ -18,6 +18,7 @@ def __init__(self, payload: dict): self.left_rate = float(payload.get('left_rate', 0.0)) self.right_rate = float(payload.get('right_rate', 0.0)) self.cam_inhib = float(payload.get('cam_inhib', 0.0)) + self.tablet_drive = dict(payload.get('tablet_drive') or {}) self.recorded_gf_left = bool(payload.get('gf_left', False)) self.recorded_gf_right = bool(payload.get('gf_right', False)) self.t = float(payload.get('t', 0.0)) @@ -52,16 +53,22 @@ def replay(trace_path: str, device: str = 'cpu') -> dict: left_driven, right_driven = [], [] gf_left_total = gf_right_total = 0 idle_frames = idle_spikes = 0 + tablet_frames = tablet_escapes = tablet_pursuits = 0 for payload in frames: recorded = FrameInput(payload) state.left_rate = recorded.left_rate state.right_rate = recorded.right_rate state.cam_inhib = recorded.cam_inhib + state.tablet_drive = recorded.tablet_drive spikes = engine.step(state, recorded.t) fired_left = bool(spikes[0, engine.l_motor_idx].any()) fired_right = bool(spikes[0, engine.r_motor_idx].any()) + if recorded.tablet_drive: + tablet_frames += 1 + tablet_escapes += int(bool(spikes[0, engine.l_escape_idx].any() or spikes[0, engine.r_escape_idx].any())) + tablet_pursuits += int(bool(spikes[0, engine.explore_idx].any())) gf_left_total += int(fired_left) gf_right_total += int(fired_right) @@ -86,6 +93,9 @@ def replay(trace_path: str, device: str = 'cpu') -> dict: 'reflex_latency_frames': sorted(latencies), 'idle_frames': idle_frames, 'idle_false_alarms': idle_spikes, + 'tablet_frames': tablet_frames, + 'tablet_escape_frames': tablet_escapes, + 'tablet_explore_frames': tablet_pursuits, } @@ -115,6 +125,11 @@ def report(result: dict) -> str: 'Idle frames : %d' % result['idle_frames'], 'Spikes with nothing on screen: %d' % result['idle_false_alarms'], '', + '--- What the tablet feed drove ---', + 'Frames with the feed driving : %d' % result.get('tablet_frames', 0), + 'Frames DNp04 answered : %d' % result.get('tablet_escape_frames', 0), + 'Frames DNp09 answered : %d' % result.get('tablet_explore_frames', 0), + '', '--- How to read this ---', 'The simulation is driven stochastically on purpose, so this run does not match the', 'recorded one number for number and is not supposed to. What holds across runs is', diff --git a/tests/test_trace_roundtrip.py b/tests/test_trace_roundtrip.py index d8034f5..82f5f64 100644 --- a/tests/test_trace_roundtrip.py +++ b/tests/test_trace_roundtrip.py @@ -6,13 +6,11 @@ import recorder import replay +from night.state import SensoryState -class FakeState: - def __init__(self, left, right, inhib): - self.left_rate = left - self.right_rate = right - self.cam_inhib = inhib +def FakeState(left, right, inhib, **tablet): + return SensoryState(left_rate=left, right_rate=right, cam_inhib=inhib, **tablet) class FakeEngine: @@ -45,6 +43,22 @@ def test_a_written_trace_reads_back_with_its_header_and_frames(): assert len(events) == 1 +def test_the_tablet_drive_survives_the_trace_and_reaches_replay(): + with tempfile.TemporaryDirectory() as folder: + path = os.path.join(folder, 'trace.jsonl') + trace = recorder.SessionRecorder(path=path) + state = FakeState(0.0, 0.0, 1000.0, camera_open=True, tablet_camera='2A', + tablet_drive={'loom_size_left': 0.4, 'loom_speed_left': 0.9}, + l_escape=True) + trace.frame(0.1, state, FakeEngine(), False, False, 0.0, 0.0, True) + trace.close() + + _header, frames, _events = recorder.load_trace(path) + assert frames[0]['tablet_camera'] == '2A' + assert frames[0]['escape_left'] is True + assert replay.FrameInput(frames[0]).tablet_drive == {'loom_size_left': 0.4, 'loom_speed_left': 0.9} + + def test_a_disabled_recorder_writes_nothing(): with tempfile.TemporaryDirectory() as folder: path = os.path.join(folder, 'trace.jsonl') @@ -87,6 +101,7 @@ def test_frame_input_reads_every_field_the_engine_needs(): def test_a_frame_missing_fields_falls_back_to_silence(): frame = replay.FrameInput({}) assert (frame.left_rate, frame.right_rate, frame.cam_inhib) == (0.0, 0.0, 0.0) + assert frame.tablet_drive == {} if __name__ == '__main__': From 9d00b4e8a399592173bee5f41a328c6b2e0e3fca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Artur=20Mendon=C3=A7a=20Arruda?= Date: Thu, 1 Oct 2026 01:52:05 -0300 Subject: [PATCH 05/14] feat(panel): show the camera feed, DNp09 and DNp04 on the timeline --- src/env/brain_view.html | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/src/env/brain_view.html b/src/env/brain_view.html index 065c073..8f196cd 100644 --- a/src/env/brain_view.html +++ b/src/env/brain_view.html @@ -595,7 +595,9 @@

FLY CONNECTOME · LIVE

in_l: frame.in_l, in_r: frame.in_r, look_l: frame.look_l, look_r: frame.look_r, eye_l: frame.eye_l || 0, eye_r: frame.eye_r || 0, gf_l: frame.gf_l, gf_r: frame.gf_r, door_l: frame.door_l, door_r: frame.door_r, - camera: frame.camera, + camera: frame.camera, cam: frame.cam, + figure: frame.figure || 0, loom: frame.loom || 0, + dnp09: frame.dnp09, dnp04_l: frame.dnp04_l, }); while (history.length && history[0].t < now - WINDOW_SECONDS - 1) history.shift(); while (timelineMarks.length && timelineMarks[0].t < now - WINDOW_SECONDS - 1) timelineMarks.shift(); @@ -603,7 +605,9 @@

FLY CONNECTOME · LIVE

el('now').textContent = (share * 100).toFixed(2) + '%'; el('fired').textContent = (frame.fired || 0).toLocaleString(); el('spikes').textContent = (frame.spikes || 0).toLocaleString(); - el('state').textContent = frame.camera ? 'TABLET UP' : ('GAZE ' + (frame.gaze || '--')); + el('state').textContent = frame.camera + ? 'TABLET UP' + (frame.cam ? ' CAM ' + frame.cam : '') + : ('GAZE ' + (frame.gaze || '--')); el('mlamp-l').classList.toggle('on', !!frame.gf_l); el('mlamp-r').classList.toggle('on', !!frame.gf_r); el('mlamp-cam').classList.toggle('on', !!frame.camera); @@ -623,6 +627,9 @@

FLY CONNECTOME · LIVE

{ label: 'DNp01 R', kind: 'gf', side: 'r' }, { label: 'DOOR R', kind: 'door', side: 'r' }, { label: 'TABLET', kind: 'tablet' }, + { label: 'CAM FEED', kind: 'feed' }, + { label: 'DNp09', kind: 'spike', key: 'dnp09' }, + { label: 'DNp04 L', kind: 'spike', key: 'dnp04_l' }, { label: 'WHOLE BRAIN', kind: 'brain' }, ]; @@ -672,7 +679,16 @@

FLY CONNECTOME · LIVE

} else if (row.kind === 'door') { if (f['door_' + row.side]) { tctx.fillStyle = 'rgba(255,92,74,0.35)'; tctx.fillRect(x, y + 2, dx, inner); } } else if (row.kind === 'tablet') { - if (f.camera) { tctx.fillStyle = 'rgba(168,85,247,0.45)'; tctx.fillRect(x, y + 2, dx, inner); } + if (f.camera) { tctx.fillStyle = f.cam === '2A' ? 'rgba(236,72,153,0.5)' : 'rgba(168,85,247,0.45)'; tctx.fillRect(x, y + 2, dx, inner); } + } else if (row.kind === 'feed') { + const level = Math.max(f.figure, f.loom); + if (level > 0) { + const h = inner * Math.min(1, level); + tctx.fillStyle = f.loom >= f.figure ? 'rgba(255,92,74,0.85)' : 'rgba(245,185,74,0.85)'; + tctx.fillRect(x, y + 2 + inner - h, dx, h); + } + } else if (row.kind === 'spike') { + if (f[row.key]) { tctx.fillStyle = color('--motor'); tctx.fillRect(x, y + 1, Math.max(dx, 2), rowH - 2); } } else if (row.kind === 'brain') { const h = inner * Math.min(1, f.share / peakShare); tctx.fillStyle = 'rgba(74,158,255,0.75)'; From ccbff3a5e88fa6644a4b387d9db10ae20a3751d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Artur=20Mendon=C3=A7a=20Arruda?= Date: Thu, 1 Oct 2026 01:54:59 -0300 Subject: [PATCH 06/14] feat(scripts): measure the tablet pathways and search between cell types --- src/graph_search.py | 15 +++-- src/scripts/debug_tablet.py | 50 ++++++++++++++++ src/scripts/fetch_data.py | 6 +- src/scripts/find_motor_pathway.py | 62 +++++++++++-------- src/scripts/measure_tablet_pathways.py | 82 ++++++++++++++++++++++++++ src/scripts/mine_inhibitors.py | 15 +++-- src/scripts/verify_connectome.py | 5 +- 7 files changed, 187 insertions(+), 48 deletions(-) create mode 100644 src/scripts/debug_tablet.py create mode 100644 src/scripts/measure_tablet_pathways.py diff --git a/src/graph_search.py b/src/graph_search.py index 6e277f5..2b5c7b0 100644 --- a/src/graph_search.py +++ b/src/graph_search.py @@ -1,6 +1,5 @@ import heapq import pandas as pd -import numpy as np from collections import defaultdict from typing import List, Dict, Optional, Tuple @@ -12,12 +11,12 @@ def load_excitatory_graph(conn_path: str) -> Dict[int, List[Tuple[float, int]]]: excitatory_edges = df[df['Excitatory x Connectivity'] > 0] graph = defaultdict(list) - for _, row in excitatory_edges.iterrows(): - src = int(row['Presynaptic_Index']) - dst = int(row['Postsynaptic_Index']) - weight = float(row['Excitatory x Connectivity']) - cost = 1.0 / weight - graph[src].append((cost, dst)) + for src, dst, weight in zip( + excitatory_edges['Presynaptic_Index'].to_numpy(), + excitatory_edges['Postsynaptic_Index'].to_numpy(), + excitatory_edges['Excitatory x Connectivity'].to_numpy(), + ): + graph[int(src)].append((1.0 / float(weight), int(dst))) return graph @@ -66,4 +65,4 @@ def _reconstruct_path(prev: Dict[int, Optional[int]], target: int) -> List[int]: while node is not None: path.append(node) node = prev[node] - return list(reversed(path)) \ No newline at end of file + return list(reversed(path)) diff --git a/src/scripts/debug_tablet.py b/src/scripts/debug_tablet.py new file mode 100644 index 0000000..9adf696 --- /dev/null +++ b/src/scripts/debug_tablet.py @@ -0,0 +1,50 @@ +import argparse +import os +import sys +import time + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) + +import config +from env.overlay import anchor_to_game +from env.tablet_feed import TabletFeed +from night.state import SensoryState +from night.tablet.senses import TabletSenses + + +def main() -> int: + parser = argparse.ArgumentParser( + description='Live tablet levels for one camera, to fit the spans in TabletVision.', + ) + parser.add_argument('camera', choices=[button.name for button in config.TABLET_VISION.cameras]) + parser.add_argument('--seconds', type=float, default=60.0) + args = parser.parse_args() + + anchor_to_game() + feed = TabletFeed() + feed.activate() + input(f'raise the tablet on CAM {args.camera} with nothing in view, then press Enter ') + if not feed.capture_reference(args.camera): + print('no frames arrived from the tablet patch') + return 1 + + state = SensoryState(camera_open=True, tablet_camera=args.camera) + senses = TabletSenses(feed) + print('watching, bring something into view. Ctrl+C stops.') + deadline = time.time() + args.seconds + try: + while time.time() < deadline: + now = time.time() + contrast = feed.contrast(args.camera) or 0.0 + levels = senses.read(state, now) + shown = ' '.join(f'{name} {level:.2f}' for name, level in levels.items()) + print(f'\rcontrast {contrast:8.0f} noise {feed.noise(args.camera):6.0f} {shown} ', end='') + time.sleep(0.1) + except KeyboardInterrupt: + pass + print() + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/src/scripts/fetch_data.py b/src/scripts/fetch_data.py index 1c47c7f..51d8d71 100644 --- a/src/scripts/fetch_data.py +++ b/src/scripts/fetch_data.py @@ -171,11 +171,7 @@ def status() -> list: ('connectivity parquet', config.CONNECTIVITY_PARQUET, _have(config.CONNECTIVITY_PARQUET)), ('completeness csv', config.COMPLETENESS_CSV, _have(config.COMPLETENESS_CSV)), ('soma coordinates', SOMA_CSV, soma_file_is_current()), - ('flywire annotations', os.path.join( - ANNOTATIONS, 'supplemental_files', 'Supplemental_file1_neuron_annotations.tsv', - ), _have(os.path.join( - ANNOTATIONS, 'supplemental_files', 'Supplemental_file1_neuron_annotations.tsv', - ))), + ('flywire annotations', config.ANNOTATIONS_TSV, _have(config.ANNOTATIONS_TSV)), ] diff --git a/src/scripts/find_motor_pathway.py b/src/scripts/find_motor_pathway.py index 4c03b7e..542df64 100644 --- a/src/scripts/find_motor_pathway.py +++ b/src/scripts/find_motor_pathway.py @@ -1,36 +1,52 @@ -import sys +import argparse import os +import sys sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) import config +from graph_search import dijkstra, load_excitatory_graph +from neural.cell_types import CellTypeIndex from neural.data_loader import get_hash_tables -from graph_search import load_excitatory_graph, find_true_sensory_nodes, find_sensory_cluster - - -def run(): - flyid2i, i2flyid = get_hash_tables(config.COMPLETENESS_CSV) - graph, nodes_with_incoming = load_excitatory_graph(config.CONNECTIVITY_PARQUET) - print(f'graph built, unique source nodes: {len(graph)}') - - sensory_candidates = find_true_sensory_nodes(graph, nodes_with_incoming, min_out_degree=5) - print(f'sensory candidates: {len(sensory_candidates)}') - motor_root_id = config.MOTOR_NEURONS.dnp01_giant_fiber[0] - motor_index = flyid2i[motor_root_id] - cluster = find_sensory_cluster(graph, sensory_candidates, motor_index, cluster_size=50, max_hops=15) +def _indices(cell_index: CellTypeIndex, flyid2i: dict, cell_type: str, side: str) -> list: + return [flyid2i[root] for root in cell_index.root_ids([cell_type], side) if root in flyid2i] - if not cluster: - print('no sensory candidate reached the motor neuron within hop limit') - return - root_ids = [i2flyid[idx] for _, idx in cluster] - print(f'top {len(cluster)} sensory cluster: {root_ids}') - - for rank, (cost, idx) in enumerate(cluster): - print(f'rank {rank+1:02d}: index={idx} root_id={i2flyid[idx]} cost={cost:.6f}') +def run(source_type: str, target_type: str, side: str, max_hops: int) -> int: + cell_index = CellTypeIndex.load() + flyid2i, i2flyid = get_hash_tables(config.COMPLETENESS_CSV) + sources = _indices(cell_index, flyid2i, source_type, side) + targets = _indices(cell_index, flyid2i, target_type, side) + if not sources or not targets: + print(f'{source_type} or {target_type} has no {side} neurons in this release') + return 1 + + print(f'{len(sources)} {source_type} and {len(targets)} {target_type} on the {side}, building the graph') + found = dijkstra(load_excitatory_graph(config.CONNECTIVITY_PARQUET), sources, targets, max_hops) + if found is None: + print(f'no excitatory path within {max_hops} hops') + return 1 + + cost, path = found + print(f'cheapest excitatory path, cost {cost:.4f} over {len(path) - 1} synapses') + for hop, index in enumerate(path): + print(f' {hop}: {cell_index.describe(i2flyid[index])}') + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser( + description='Cheapest excitatory path between two FlyWire cell types, weights as cost.', + ) + parser.add_argument('--source', default='LPLC2') + parser.add_argument('--target', default='DNp01') + parser.add_argument('--side', default='left', choices=('left', 'right')) + parser.add_argument('--max-hops', type=int, default=10) + args = parser.parse_args() + return run(args.source, args.target, args.side, args.max_hops) if __name__ == '__main__': - run() \ No newline at end of file + raise SystemExit(main()) diff --git a/src/scripts/measure_tablet_pathways.py b/src/scripts/measure_tablet_pathways.py new file mode 100644 index 0000000..34679c1 --- /dev/null +++ b/src/scripts/measure_tablet_pathways.py @@ -0,0 +1,82 @@ +import argparse +import os +import statistics +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) + +import config +from night.engine import ConnectomeEngine +from night.state import SensoryState + +READOUTS = ('giant_fiber_left', 'looming_escape_left', 'explore') +LEVELS = (0.1, 0.25, 0.5, 1.0) + + +def conditions() -> list: + rate = config.SIMULATION_PARAMS.base_sensory_rate_hz + rows = [ + ('idle, tablet down', {}, 0.0, 0.0), + ('idle, tablet up', {}, rate, 0.0), + ('hallway eye, tablet down', {}, 0.0, 1.0), + ] + for level in LEVELS: + rows.append((f'loom {level:.2f}, tablet up', + {'loom_size_left': level, 'loom_speed_left': level}, rate, 0.0)) + for level in LEVELS: + rows.append((f'figure {level:.2f}, tablet up', {'figure_left': level}, rate, 0.0)) + return rows + + +def trial(engine: ConnectomeEngine, state: SensoryState, frames: int) -> dict: + engine.reset() + first = dict.fromkeys(READOUTS) + for frame in range(1, frames + 1): + spikes = engine.step(state, 0.0)[0] + for name in READOUTS: + if first[name] is None and spikes[engine.neurons.outputs[name]].any(): + first[name] = frame + return first + + +def measure(engine: ConnectomeEngine, drive: dict, inhibitors_hz: float, eye: float, + frames: int, trials: int) -> dict: + state = SensoryState(left_rate=eye, cam_inhib=inhibitors_hz, tablet_drive=drive) + results = [trial(engine, state, frames) for _ in range(trials)] + summary = {} + for name in READOUTS: + hits = [result[name] for result in results if result[name] is not None] + latency = statistics.median(hits) if hits else None + summary[name] = (len(hits), latency) + return summary + + +def cell(hits: int, latency, trials: int) -> str: + shown = '-' if latency is None else f'{latency:g}' + return f'{hits}/{trials} @ {shown}' + + +def main() -> int: + parser = argparse.ArgumentParser( + description='Which descending neuron answers each tablet channel, measured on the running engine.', + ) + parser.add_argument('--frames', type=int, default=40) + parser.add_argument('--trials', type=int, default=6) + parser.add_argument('--device', default='cpu', choices=('cpu', 'cuda')) + args = parser.parse_args() + + engine = ConnectomeEngine(args.device) + print('populations:', ', '.join(f'{k} {v}' for k, v in engine.neurons.sizes().items())) + print(f'{args.trials} trials of {args.frames} frames each, cell shows trials answered ' + f'and median first-spike frame\n') + header = f'{"condition":<28}' + ''.join(f'{name:>24}' for name in READOUTS) + print(header) + for label, drive, inhibitors_hz, eye in conditions(): + summary = measure(engine, drive, inhibitors_hz, eye, args.frames, args.trials) + cells = ''.join(f'{cell(*summary[name], args.trials):>24}' for name in READOUTS) + print(f'{label:<28}{cells}', flush=True) + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/src/scripts/mine_inhibitors.py b/src/scripts/mine_inhibitors.py index 26187bf..71f557f 100644 --- a/src/scripts/mine_inhibitors.py +++ b/src/scripts/mine_inhibitors.py @@ -5,18 +5,17 @@ import pandas as pd import config - -FLYWIRE_ANNOTATIONS = os.path.join( - os.path.dirname(config.PROJECT_ROOT), - 'flywire_annotations', 'supplemental_files', 'Supplemental_file1_neuron_annotations.tsv' -) +from neural.cell_types import CellTypeIndex def mine_top_50(): conn = pd.read_parquet(config.CONNECTIVITY_PARQUET) - anno = pd.read_csv(FLYWIRE_ANNOTATIONS, sep='\t', low_memory=False) + anno = pd.read_csv(config.ANNOTATIONS_TSV, sep='\t', low_memory=False) - left_gf, right_gf = config.MOTOR_NEURONS.dnp01_giant_fiber + cell_index = CellTypeIndex.load() + giant_fiber = config.CELL_POPULATIONS.giant_fiber + left_gf = cell_index.root_ids(giant_fiber, 'left')[0] + right_gf = cell_index.root_ids(giant_fiber, 'right')[0] gaba_ids = set(anno[anno['top_nt'] == 'gaba']['root_id'].unique()) @@ -38,4 +37,4 @@ def mine_top_50(): if __name__ == '__main__': - mine_top_50() \ No newline at end of file + mine_top_50() diff --git a/src/scripts/verify_connectome.py b/src/scripts/verify_connectome.py index 571b441..aee1078 100644 --- a/src/scripts/verify_connectome.py +++ b/src/scripts/verify_connectome.py @@ -7,10 +7,7 @@ import config from brain_adapter import BrainAdapter -FLYWIRE_ANNOTATIONS = os.path.join( - os.path.dirname(config.PROJECT_ROOT), - 'flywire_annotations', 'supplemental_files', 'Supplemental_file1_neuron_annotations.tsv' -) +FLYWIRE_ANNOTATIONS = config.ANNOTATIONS_TSV PROOFREAD_783 = 139255 From 1984a08183fe1bd150474e6adbefbfd3b07889d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Artur=20Mendon=C3=A7a=20Arruda?= Date: Thu, 1 Oct 2026 01:57:19 -0300 Subject: [PATCH 07/14] docs: record the cell type populations and the tablet watch on Foxy --- README.md | 33 +++-- ...5-populations-are-resolved-by-cell-type.md | 35 ++++++ ...the-lobula-and-dnp04-answers-through-it.md | 44 +++++++ ...foxy-was-never-on-the-screen-it-watched.md | 114 ++++++++++++++++++ 4 files changed, 218 insertions(+), 8 deletions(-) create mode 100644 docs/adr/0025-populations-are-resolved-by-cell-type.md create mode 100644 docs/adr/0026-the-tablet-feeds-the-lobula-and-dnp04-answers-through-it.md create mode 100644 docs/experiments/13-foxy-was-never-on-the-screen-it-watched.md diff --git a/README.md b/README.md index 364244f..a35f370 100644 --- a/README.md +++ b/README.md @@ -11,15 +11,17 @@ https://www.youtube.com/watch?v=4UNPlA-YJtw. ## What the fly actually does -Three behaviours come out of the network, and each one is a different pathway. +Four behaviours come out of the network, and each one is a different pathway. -**Slamming a door.** The hallway is captured while the light is on and compared against a reference of that same hallway empty. A difference above threshold drives the corresponding eye cluster at full rate, that excitation propagates through the real connectivity, and when DNp01, the Giant Fiber, crosses its own firing threshold the door closes. Measured on this machine, the neuron needs seven to nine engine frames of sustained input to answer, which is why the light is held for twelve. +**Slamming a door.** The hallway is captured while the light is on and compared against a reference of that same hallway empty. A difference above threshold drives the corresponding eye cluster at full rate, every LPLC2 and LC4 neuron on that side, that excitation propagates through the real connectivity, and when DNp01, the Giant Fiber, crosses its own firing threshold the door closes. Measured on this machine, the neuron needs seven to nine engine frames of sustained input to answer, which is why the light is held for twelve. **Checking a hallway.** The difference in membrane potential between the left and right eye populations is a slow spontaneous signal the network produces on its own, with a lag-1 autocorrelation of 0.97. It is accumulated to a bound together with what the last look revealed and a per-side habituation term, and whichever side wins gets looked at. There is one starvation guard, a clock that forces a look at a hallway left unwatched for thirty seconds, and every look it causes is counted separately in the session report so its share stays visible. **Raising the monitor.** DNp09's own subthreshold membrane potential is accumulated the same way. A real spike raises the tablet outright, and the tablet comes down when that drive fades rather than when a timer expires. -Raising the tablet also drives the GABAergic inhibitor clusters at full rate, which silences both descending neurons. The fly is genuinely blind while it watches the cameras, exactly as it would be in the game. +Raising the tablet also drives the GABAergic inhibitor clusters at full rate, which silences the Giant Fiber, so no door reflex can fire through the tablet, exactly as no door button can be pressed in the game. + +**Watching for Foxy.** The tablet always comes up on Pirate Cove, CAM 1C. What differs from the cove as it looked at the start of the night drives LC9 and LC31a, the lobula columns that feed DNp09 and are used by flies to pursue a moving object. When DNp09 fires, the gaze follows to the west hall, CAM 2A. There the size of whatever differs drives LPLC2 and its growth drives LC4, and DNp04, a looming escape neuron the inhibitors barely reach, fires on it. The fly then drops the tablet and closes the left door. Measured on the engine, each channel wakes its own descending neuron and never the other, and the Giant Fiber stays silent through both (ADR 0026, experiment 13). ## What this is not @@ -53,7 +55,7 @@ The simulation needs roughly 140 MB of FlyWire data that is deliberately not com │ └── brain_mesh_flywire.ply 1 MB, brain outline for the 3D panel ├── flywire_annotations/ git clone https://github.com/flyconnectome/flywire_annotations │ └── supplemental_files/ -│ └── Supplemental_file1_neuron_annotations.tsv 31 MB, super_class per neuron +│ └── Supplemental_file1_neuron_annotations.tsv 31 MB, cell type, side and super_class per neuron └── Fly-NAF/ this repository ``` @@ -135,6 +137,9 @@ each region actually reads. - `src/scripts/vision_calibrator.py` writes the measured values into `logs/`. - `src/scripts/debug_vision.py` and `src/scripts/debug_camera.py` show live what each capture region is reading. - `src/scripts/debug_reflex.py` shows the live contrast of both hallways against the threshold, and says whether a door would slam. Run it with a light held on, because a reading taken with the light off means nothing. +- `src/scripts/debug_tablet.py 1C` or `2A` records one camera with nothing in view and then prints the live drive it would send into the brain. That is how the spans in `TabletVision` are fitted, and how the CAM 2A button position, inferred from CAM 4B, is confirmed. + +The camera references are not cached. At the start of every night the fly raises the tablet once, records a few seconds of CAM 1C and CAM 2A while the curtain is closed and the hall is empty, and lowers it again, which also stalls Foxy while it happens. The eye references are captured once and cached in `logs/vision_reference/`, alongside a fingerprint of the layout they were taken on. They are re-measured @@ -206,6 +211,13 @@ uv run python src/scripts/verify_connectome.py It prints the neuron count, the edge count, the coverage against the FlyWire annotations, the super class of every neuron left out, the dimensions of the matrix handed to the engine, how many rows were dropped before it, and the gain in force. It needs the connectome data in place and nothing else. +```bash +uv run python src/scripts/measure_tablet_pathways.py +uv run python src/scripts/find_motor_pathway.py --source LC9 --target DNp09 +``` + +The first drives each tablet channel into the running engine with the inhibitors on and prints which descending neuron answers and on which frame, the table ADR 0026 rests on. The second prints the cheapest excitatory path between any two FlyWire cell types. + Every run also writes a trace to `logs/traces/`, one line per engine frame, carrying the input the eyes produced and what the descending neurons did with it. `uv run python src/replay.py ` feeds that input back through a real @@ -228,9 +240,10 @@ uv run python tests/test_hallway_detection.py ``` Most of them stub out the vision and motor layers and finish in under a second. -Four of them, `test_inhibition.py`, `test_pathways.py`, `test_engine_rate_reset.py` -and `test_search_loop_wiring.py`, load the real connectome, so they need the -FlyWire data in place and take a few minutes each. +`test_search_loop_wiring.py` plays scripted nights in real time and takes about +two minutes. Two of them, `test_inhibition.py` and `test_pathways.py`, run the +real engine on the full connectome, so they need the FlyWire data in place and +take several minutes each. ## Documentation @@ -242,7 +255,10 @@ targets that follow the game window, ADR 0019 covers the trace format and what replay does and does not prove, ADR 0020 covers which settings are exposed for tuning and where their ranges come from, ADR 0021 covers the launcher and why the game is measured only once it is in front, and ADR 0022 covers the -panel and the rule that only simulation variables drive it. +panel and the rule that only simulation variables drive it. ADR 0025 covers how +populations are named by cell type rather than pasted as ids, and ADR 0026 and +experiment 13 cover the tablet, Foxy and why DNp04 rather than the Giant Fiber +answers through it. ## Licensing and credits @@ -276,3 +292,4 @@ Five Nights at Freddy's is by Scott Cawthon and is not affiliated with this proj | 1.13 | Described the one click setup and the git and zip routes for the data | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-30 | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-30 | | 1.14 | Named run.py as the only entry point | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-30 | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-30 | | 1.15 | Pointed to the night package that main now wires | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-30 | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-30 | +| 1.16 | Described the tablet watch on Foxy, the cell type populations and the new scripts, and corrected which tests are slow | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-30 | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-30 | diff --git a/docs/adr/0025-populations-are-resolved-by-cell-type.md b/docs/adr/0025-populations-are-resolved-by-cell-type.md new file mode 100644 index 0000000..b94eb88 --- /dev/null +++ b/docs/adr/0025-populations-are-resolved-by-cell-type.md @@ -0,0 +1,35 @@ +# ADR 0025: Populations Are Resolved by FlyWire Cell Type + +## Status +Accepted. Extends ADR 0005, whose rule that every pathway is a real FlyWire connection stands. + +## Context +Every population the engine drove or read was a list of root ids pasted into `config.py`. The eye clusters were fifty LPLC2 per side, chosen in experiment 03, out of the 108 and 102 that FlyWire 783 annotates. LC4, the other looming projection onto the Giant Fiber, was absent even though it contributes 374 and 431 synapses to DNp01 against 458 and 622 from LPLC2. Von Reyn et al. 2017 and Ache et al. 2019 show that the Giant Fiber sums the two, LC4 encoding angular velocity and LPLC2 angular size. + +Pasted ids also hide intent. A list of fifty integers cannot say that it means "LPLC2 on the left", so nothing flags it when half the type is missing, and adding a population for a new behaviour means another pasted list. + +## Decision +Populations are declared by role and cell type in `CellPopulations`, and `neural/cell_types.py` resolves them against the Schlegel et al. 2024 annotation table, split by the `side` column. The resolved table is cached next to the connectome as `cell_types_783.parquet` and rebuilt whenever the annotation file is newer. + +| Role | Cell types | Neurons, left and right | +|---|---|---| +| Hallway eye | LPLC2, LC4 | 162, 152 | +| Figure | LC9, LC31a | 105, 109 | +| Loom size | LPLC2 | 108, 102 | +| Loom speed | LC4 | 54, 50 | +| Door reflex | DNp01 | 1, 1 | +| Looming escape | DNp04 | 1, 1 | +| Exploration | DNp09 | 2 | + +`night/neurons.py` turns the roles into a `NeuronMap` of engine indices, and the engine builds its rates from that map rather than from named fields. The GABAergic inhibitor lists stay as root ids, because they are the output of a mining script over synapse counts, not a cell type. + +## Consequences +- The hallway eye now carries all of LPLC2 and all of LC4. Measured on the engine, the Giant Fiber still answers at frame 3, and DNp02 and DNp11 are recruited alongside it, as the looming literature predicts. +- The annotation table becomes a runtime dependency. It was already a required download in `fetch_data.py`, and the first run pays a few seconds to build the cache. +- A population that resolves to nothing raises at start-up, so a renamed cell type in a future release fails loudly instead of leaving a silent cluster. +- The search drive normalises its evidence by the larger of the two eye populations, since the sides are no longer the same size. +- `find_motor_pathway.py`, broken since the helpers it imported were removed, now searches between any two cell types. + +| Version | Description | Author(s) | Date | Reviewer(s) | Review Date | +|---|---|---|---|---|---| +| 1.0 | Populations declared by cell type and resolved from the FlyWire annotations, LC4 added to the eye | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-30 | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-30 | diff --git a/docs/adr/0026-the-tablet-feeds-the-lobula-and-dnp04-answers-through-it.md b/docs/adr/0026-the-tablet-feeds-the-lobula-and-dnp04-answers-through-it.md new file mode 100644 index 0000000..93bbaf9 --- /dev/null +++ b/docs/adr/0026-the-tablet-feeds-the-lobula-and-dnp04-answers-through-it.md @@ -0,0 +1,44 @@ +# ADR 0026: The Tablet Feeds the Lobula, and DNp04 Answers Through It + +## Status +Accepted. Resolves the open point left by ADR 0014, that the tablet decided only whether to look and never which camera. + +## Context +Foxy is only visible through the tablet. He advances from Pirate Cove (CAM 1C) while no camera is in use, runs down the west hall (CAM 2A) to the left door once he leaves, and any camera stalls him. The fly raised the tablet on whatever camera the game last showed, and nothing on that screen reached a neuron, so Foxy could not be perceived at all. + +The door reflex cannot be reused through the tablet. ADR 0006 gates the Giant Fiber with GABAergic inhibitors while the tablet is up, and those inhibitors supply 35 percent of DNp01's input. Among the other looming descending neurons, DNp04 receives 1,671 synapses from LC4 and LPLC2 on the left and only 6.8 percent of its input from the same inhibitors. DNp09, which already raises the tablet, takes its largest visual input from LC9 and LC31a, lobula columns tuned to fly sized objects and used in pursuit (Bidaye et al. 2020, Klapoetke et al. 2022). + +The table below was measured on the running engine with the inhibitors at full drive, six trials of forty frames each. + +``` +condition, tablet up DNp01 left DNp04 left DNp09 +nothing on screen 0/6 0/6 0/6 +loom 0.10 0/6 0/6 0/6 +loom 0.25 to 1.00 0/6 6/6 @ 3 0/6 +figure 0.10 0/6 0/6 2/6 @ 4 +figure 0.25 to 1.00 0/6 0/6 6/6 @ 3 +``` + +## Decision +The tablet reads the west side through two lobula channels, chosen by the camera being watched. + +- Raising the tablet taps CAM 1C. This is a fixed motor gesture, recorded as such, in the same class as the pan delay. +- The cove is compared against a bank of frames captured at the start of the night with the curtain closed. Taking the nearest frame in the bank absorbs the camera pan. The distance above the bank's own noise, over `figure_span_mse`, drives LC9 and LC31a on the left. +- A DNp09 spike while the cove is readable moves the gaze to CAM 2A, once per raise. The object has left and the fly follows it. +- In CAM 2A the same distance drives LPLC2 and its growth per second drives LC4, the size and velocity split of the door reflex. +- A DNp04 spike while the tablet is up drops the tablet, closes the door on that side, recentres the view and re-anchors the office before the lowering counts as done. + +`env/tablet_feed.py` captures and compares the feed, `night/tablet/` holds the transduction, the gaze, the per-frame watch and the start-of-night references, and the door bookkeeping moves out of the engine loop into `night/doors.py` so the Giant Fiber and DNp04 close doors through one path. + +## Consequences +- Which descending neuron answers is decided by the connectome. Neither channel reaches the other's readout, and the Giant Fiber stays gated throughout, so `test_inhibition.py` keeps its guarantee and now also asserts the dissociation. +- Seeing Foxy in the cove keeps DNp09 depolarised, which keeps the tablet up longer, which stalls him. That behaviour is not written anywhere, it follows from DNp09 already controlling the tablet. +- Bonnie standing in 2A drives LPLC2 without growth and still wakes DNp04, so the left door closes early. That is a correct defensive reading and costs power. +- The spans that convert image distance into drive are game-side constants that have not been fitted against a recorded Foxy. `debug_tablet.py` prints them live, and traces now carry the tablet drive so they can be refitted from recorded nights. +- The CAM 2A button position is inferred from CAM 4B and needs confirming on screen. +- The start of the night gains one tablet raise of about eight seconds to record the two banks. Foxy is stalled for all of it. +- The escape is a chain of motor actions, and lowering the tablet plus panning to the left door takes over a second. Whether that fits inside Foxy's run is the first thing a recorded night has to answer. + +| Version | Description | Author(s) | Date | Reviewer(s) | Review Date | +|---|---|---|---|---|---| +| 1.0 | CAM 1C drives LC9 and LC31a, DNp09 pursues to CAM 2A, CAM 2A drives LPLC2 and LC4, DNp04 closes the door | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-30 | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-30 | diff --git a/docs/experiments/13-foxy-was-never-on-the-screen-it-watched.md b/docs/experiments/13-foxy-was-never-on-the-screen-it-watched.md new file mode 100644 index 0000000..1f5b6c3 --- /dev/null +++ b/docs/experiments/13-foxy-was-never-on-the-screen-it-watched.md @@ -0,0 +1,114 @@ +# Experiment 13: Foxy Was Never On The Screen It Watched + +## What I Saw + +Night 3 is where Foxy starts moving, and the fly never once went looking for +him. The camera buttons for Pirate Cove and the east hall had been sitting in +`MotorCalibration` since the second pull request, measured and never pressed, +and at some point I deleted them because nothing read them. That should have +been the clue. My first assumption was that the tablet simply came up too +rarely, so I went back to the reports, and the pulls were fine, roughly one +every forty seconds as ADR 0014 had fitted. + +## Two Things That Were Never Wired + +The tablet came up on whatever camera the game had last shown, which on a +fresh night is the show stage. So the fly was raising the monitor and staring +at Freddy. Worse, nothing it saw there reached a neuron at all. While the +monitor was up the vision loop measured one thing, whether the monitor was up, +and used that to drive the GABAergic inhibitors. The picture itself went +nowhere. The fly was not ignoring Foxy, it had no way to see him. + +That still left the question of what the fly should see. A camera feed is not +a hallway with a light on it, and I did not want to invent a pathway. So +before writing anything I went to the connectome for what it already has for +an object that appears and one that approaches. + +## What The 783 Release Actually Contains + +The first thing I checked was the eye itself. The two clusters from +experiment 03 are LPLC2, fifty per side, and the release has 108 on the left +and 102 on the right, so the fly was looking at doors with half its looming +detectors. The other half of the looming input to the giant fiber was not +there at all. LC4 gives DNp01 374 and 431 +synapses against 458 and 622 from LPLC2, and von Reyn's work splits the two +neatly, LC4 carrying how fast the shape grows and LPLC2 how large it is. + +The second was DNp09, which raises the tablet. Its largest visual inputs are +LC9 and LC31a, 385 and 338 synapses on the left, and LC9 belongs to the +lobula columns tuned to fly sized objects that males use to pursue females. +DNp09 is the pursuit neuron from Bidaye's walking paper and the freezing +neuron from Zacarias. A neuron that means "an object is out there, go after +it" already raises the monitor, and nothing had ever shown it an object. + +## The Wall I Hit + +The obvious plan was to show the west hall to the looming populations and let +DNp01 close the door. It cannot work. With the tablet up the inhibitors are at +full drive, and 35 percent of everything DNp01 receives comes from them. +`test_inhibition.py` exists precisely to prove the giant fiber stays silent +then. I spent a while wondering whether to weaken that, which would have +meant the fly could slam a door through the tablet, which the game does not +allow either. + +What got me out was listing every looming descending neuron and asking how +much of its input the inhibitors own. DNp02 18 percent, DNp06 7.6, DNp11 4.7, +DNp04 6.8. DNp04 also takes 1,671 synapses from LC4 and LPLC2 on the left, more +than the giant fiber does. It is one of the parallel escape neurons that drive +the slower takeoff when the giant fiber does not fire first. + +## Measuring It Instead Of Believing It + +I ran the engine as it runs at night, with the inhibitors at their real rate +and their refractory period intact, six trials of forty frames each, and +recorded the frame of the first spike. + +| Condition, tablet up | DNp01 left | DNp04 left | DNp09 | +|---|---|---|---| +| Nothing on screen | 0 of 6 | 0 of 6 | 0 of 6 | +| Looming at 0.10 | 0 of 6 | 0 of 6 | 0 of 6 | +| Looming at 0.25 | 0 of 6 | 6 of 6, frame 3 | 0 of 6 | +| Looming at 1.00 | 0 of 6 | 6 of 6, frame 3 | 0 of 6 | +| Figure at 0.10 | 0 of 6 | 0 of 6 | 2 of 6, frame 4 | +| Figure at 0.25 | 0 of 6 | 0 of 6 | 6 of 6, frame 3 | + +It is a clean double dissociation. Looming wakes DNp04 and nothing else, a +figure wakes DNp09 and nothing else, and the giant fiber stays gated through +both. With the tablet down the same eye still answers the hallway at frame 3, +now through DNp01, DNp04 and DNp09 together. I did not have to choose a +threshold for any of it, because the brain chooses one somewhere between a +tenth and a quarter of the span on both channels. +`src/scripts/measure_tablet_pathways.py` reproduces the table. + +## What The Fly Does Now + +The tablet always comes up on Pirate Cove. The cove is compared against a bank +of frames recorded at the start of the night, while the curtain is still +closed, and the distance drives LC9 and LC31a on the left. When DNp09 fires +the gaze follows the object down the west hall to 2A, once per raise. In 2A +the distance drives LPLC2 and its growth drives LC4, and when DNp04 fires the +fly drops the tablet, closes the left door, comes back to centre and checks the +office before it trusts the inhibitors are off again. Which camera to open +first is a fixed gesture and I would rather say so than dress it up. + +Any camera already stalls Foxy, so the fly watching the cove longer when +something is in it is the strategy a good player uses, and here it falls out +of a neuron that was already doing the watching. + +## What Is Not Measured Yet + +Everything on the game side. The spans that turn an image distance into a +drive level, 1000 for both channels and 3000 per second for growth, are placed +so a clearly visible Foxy lands well above a quarter of the span, but I have +not recorded him. The 2A button position comes from the symmetry with 4B and +has to be checked. `src/scripts/debug_tablet.py` prints the live levels for +one camera so the spans can be fitted, and every trace now records the tablet +drive frame by frame, so a night with Foxy in it can be replayed through the +brain the same way the hallways are. + +The part I trust least is time. The run from 2A to the office is short, and +lowering the tablet, panning to the door and pressing it takes most of it. + +| Version | Description | Author(s) | Date | Reviewer(s) | Review Date | +|---|---|---|---|---|---| +| 1.0 | Why the fly never saw Foxy, the cell types behind the fix and the double dissociation that justifies it | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-30 | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-30 | From 20b7db3c396a3f61373e979166b101c8a12b1efa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Artur=20Mendon=C3=A7a=20Arruda?= Date: Thu, 1 Oct 2026 19:48:40 -0300 Subject: [PATCH 08/14] refactor: move the source into the flynaf package and run the tests under pytest --- .github/workflows/tests.yml | 19 +++++ README.md | 64 ++++++++------- ...s-the-flynaf-package-and-pytest-runs-it.md | 65 ++++++++++++++++ pyproject.toml | 28 +++++++ run.py | 19 ++--- src/{night => flynaf}/__init__.py | 0 src/{ => flynaf}/brain_adapter.py | 9 ++- src/{ => flynaf}/config.py | 3 +- src/{ => flynaf}/env/anchor.py | 0 .../env/assets/CHANGES-neuromechfly.txt | 2 +- .../env/assets/LICENSE-neuromechfly.txt | 0 .../env/assets/neuromechfly.bin.gz | Bin src/{ => flynaf}/env/brain_view.html | 0 src/{ => flynaf}/env/brain_view.py | 2 +- src/{ => flynaf}/env/calibrator.py | 3 +- src/{ => flynaf}/env/cascade.py | 0 src/{ => flynaf}/env/desktop/__init__.py | 0 src/{ => flynaf}/env/desktop/headless.py | 0 src/{ => flynaf}/env/desktop/linux.py | 0 src/{ => flynaf}/env/desktop/windows.py | 0 src/{ => flynaf}/env/input_controller.py | 7 +- src/{ => flynaf}/env/launcher.html | 0 src/{ => flynaf}/env/overlay.py | 5 +- src/{ => flynaf}/env/profiles.py | 0 src/{ => flynaf}/env/tablet_feed.py | 4 +- src/{ => flynaf}/env/vision.py | 12 ++- src/{ => flynaf}/graph_search.py | 6 +- src/{ => flynaf}/launcher.py | 8 +- src/{ => flynaf}/main.py | 31 ++++---- src/{ => flynaf}/neural/cell_types.py | 4 +- src/{ => flynaf}/neural/data_loader.py | 5 +- src/{ => flynaf}/neural/models.py | 4 +- .../tablet => flynaf/night}/__init__.py | 0 src/{ => flynaf}/night/calibration.py | 12 +-- src/{ => flynaf}/night/doors.py | 4 +- src/{ => flynaf}/night/engine.py | 10 +-- src/{ => flynaf}/night/monitor.py | 12 +-- src/{ => flynaf}/night/motor.py | 0 src/{ => flynaf}/night/neurons.py | 2 +- src/{ => flynaf}/night/panel.py | 8 +- src/{ => flynaf}/night/state.py | 0 src/flynaf/night/tablet/__init__.py | 0 src/{ => flynaf}/night/tablet/gaze.py | 0 src/{ => flynaf}/night/tablet/references.py | 6 +- src/{ => flynaf}/night/tablet/senses.py | 6 +- src/{ => flynaf}/night/tablet/transduction.py | 0 src/{ => flynaf}/night/tablet/watch.py | 6 +- src/{ => flynaf}/night/tasks.py | 30 +++---- src/{ => flynaf}/recorder.py | 2 +- src/{ => flynaf}/replay.py | 11 +-- src/{ => flynaf}/scripts/build_fly_model.py | 7 +- src/{ => flynaf}/scripts/debug_camera.py | 11 ++- src/{ => flynaf}/scripts/debug_reflex.py | 12 +-- src/{ => flynaf}/scripts/debug_tablet.py | 14 ++-- src/{ => flynaf}/scripts/debug_vision.py | 9 +-- src/{ => flynaf}/scripts/fetch_data.py | 5 +- .../scripts/find_motor_pathway.py | 12 +-- .../scripts/measure_tablet_pathways.py | 10 +-- src/{ => flynaf}/scripts/mine_inhibitors.py | 10 +-- src/{ => flynaf}/scripts/select_roi.py | 5 +- src/{ => flynaf}/scripts/verify_connectome.py | 8 +- src/{ => flynaf}/scripts/vision_calibrator.py | 10 +-- src/{ => flynaf}/search_drive.py | 2 +- src/{ => flynaf}/telemetry.py | 4 +- src/{ => flynaf}/tuning.py | 2 +- tests/conftest.py | 14 ++++ tests/test_brain_view_geometry.py | 9 +-- tests/test_camera_reference_refresh.py | 6 +- tests/test_cascade.py | 6 +- tests/test_cell_types.py | 11 +-- tests/test_corollary_discharge.py | 10 +-- tests/test_door_idempotency.py | 8 +- tests/test_door_release.py | 7 +- tests/test_engine_rate_reset.py | 12 +-- tests/test_explore_drive.py | 9 +-- tests/test_fetch_data.py | 5 +- tests/test_focus_handback.py | 6 +- tests/test_hallway_detection.py | 8 +- tests/test_inhibition.py | 12 +-- tests/test_inspection_window.py | 10 +-- tests/test_launcher.py | 6 +- tests/test_monitor_lowering.py | 13 +--- tests/test_motor_pan_settle.py | 8 +- tests/test_motor_suppression.py | 7 -- tests/test_overlay_placement.py | 11 +-- tests/test_pathways.py | 12 +-- tests/test_saccade_scheduling.py | 14 ++-- tests/test_search_loop_wiring.py | 12 +-- tests/test_startup_readiness.py | 10 +-- tests/test_tablet_escape.py | 12 +-- tests/test_tablet_gaze.py | 15 ++-- tests/test_tablet_senses.py | 9 +-- tests/test_tablet_transduction.py | 9 +-- tests/test_trace_roundtrip.py | 8 +- tests/test_tuning_bounds.py | 6 +- tests/test_vision_buffers.py | 5 +- tests/test_vision_reference_cache.py | 6 +- uv.lock | 73 +++++++++++++++++- 98 files changed, 470 insertions(+), 429 deletions(-) create mode 100644 .github/workflows/tests.yml create mode 100644 docs/adr/0027-the-source-is-the-flynaf-package-and-pytest-runs-it.md rename src/{night => flynaf}/__init__.py (100%) rename src/{ => flynaf}/brain_adapter.py (93%) rename src/{ => flynaf}/config.py (98%) rename src/{ => flynaf}/env/anchor.py (100%) rename src/{ => flynaf}/env/assets/CHANGES-neuromechfly.txt (94%) rename src/{ => flynaf}/env/assets/LICENSE-neuromechfly.txt (100%) rename src/{ => flynaf}/env/assets/neuromechfly.bin.gz (100%) rename src/{ => flynaf}/env/brain_view.html (100%) rename src/{ => flynaf}/env/brain_view.py (99%) rename src/{ => flynaf}/env/calibrator.py (99%) rename src/{ => flynaf}/env/cascade.py (100%) rename src/{ => flynaf}/env/desktop/__init__.py (100%) rename src/{ => flynaf}/env/desktop/headless.py (100%) rename src/{ => flynaf}/env/desktop/linux.py (100%) rename src/{ => flynaf}/env/desktop/windows.py (100%) rename src/{ => flynaf}/env/input_controller.py (98%) rename src/{ => flynaf}/env/launcher.html (100%) rename src/{ => flynaf}/env/overlay.py (99%) rename src/{ => flynaf}/env/profiles.py (100%) rename src/{ => flynaf}/env/tablet_feed.py (98%) rename src/{ => flynaf}/env/vision.py (99%) rename src/{ => flynaf}/graph_search.py (97%) rename src/{ => flynaf}/launcher.py (98%) rename src/{ => flynaf}/main.py (75%) rename src/{ => flynaf}/neural/cell_types.py (96%) rename src/{ => flynaf}/neural/data_loader.py (99%) rename src/{ => flynaf}/neural/models.py (99%) rename src/{night/tablet => flynaf/night}/__init__.py (100%) rename src/{ => flynaf}/night/calibration.py (89%) rename src/{ => flynaf}/night/doors.py (96%) rename src/{ => flynaf}/night/engine.py (92%) rename src/{ => flynaf}/night/monitor.py (96%) rename src/{ => flynaf}/night/motor.py (100%) rename src/{ => flynaf}/night/neurons.py (98%) rename src/{ => flynaf}/night/panel.py (97%) rename src/{ => flynaf}/night/state.py (100%) create mode 100644 src/flynaf/night/tablet/__init__.py rename src/{ => flynaf}/night/tablet/gaze.py (100%) rename src/{ => flynaf}/night/tablet/references.py (93%) rename src/{ => flynaf}/night/tablet/senses.py (89%) rename src/{ => flynaf}/night/tablet/transduction.py (100%) rename src/{ => flynaf}/night/tablet/watch.py (94%) rename src/{ => flynaf}/night/tasks.py (93%) rename src/{ => flynaf}/recorder.py (99%) rename src/{ => flynaf}/replay.py (96%) rename src/{ => flynaf}/scripts/build_fly_model.py (97%) rename src/{ => flynaf}/scripts/debug_camera.py (93%) rename src/{ => flynaf}/scripts/debug_reflex.py (90%) rename src/{ => flynaf}/scripts/debug_tablet.py (84%) rename src/{ => flynaf}/scripts/debug_vision.py (90%) rename src/{ => flynaf}/scripts/fetch_data.py (98%) rename src/{ => flynaf}/scripts/find_motor_pathway.py (87%) rename src/{ => flynaf}/scripts/measure_tablet_pathways.py (94%) rename src/{ => flynaf}/scripts/mine_inhibitors.py (88%) rename src/{ => flynaf}/scripts/select_roi.py (92%) rename src/{ => flynaf}/scripts/verify_connectome.py (95%) rename src/{ => flynaf}/scripts/vision_calibrator.py (96%) rename src/{ => flynaf}/search_drive.py (99%) rename src/{ => flynaf}/telemetry.py (99%) rename src/{ => flynaf}/tuning.py (99%) create mode 100644 tests/conftest.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..13d24bf --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,19 @@ +name: tests + +on: + push: + branches: [main] + pull_request: + +jobs: + test: + runs-on: windows-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v5 + - uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + - run: uv sync --locked + - run: uv run ruff check src run.py tests + - run: uv run pytest -q -rs diff --git a/README.md b/README.md index a35f370..87e0113 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # Fly-NAF +[![tests](https://github.com/ArtyMend07/Fly-NAF/actions/workflows/tests.yml/badge.svg)](https://github.com/ArtyMend07/Fly-NAF/actions/workflows/tests.yml) + A whole-brain *Drosophila melanogaster* connectome simulation that plays Five Nights at Freddy's 1. 138,639 of the 139,255 proofread neurons in the FlyWire 783 connectome run as leaky integrate-and-fire units, wired by the measured synaptic weights. That is 99.56% of the brain, and the ones left out are almost all sensory afferents rather than interneurons. Nothing else is subset. The screen is fed into the fly's visual clusters, and the mouse is driven by reading its descending neurons. @@ -25,9 +27,9 @@ Raising the tablet also drives the GABAergic inhibitor clusters at full rate, wh ## What this is not -There is no policy, no reward, no training and no learning whatsoever. The connectome is fixed at what FlyWire measured. Every constant in `src/config.py` was fitted against measurements of the simulated network rather than chosen by feel. +There is no policy, no reward, no training and no learning whatsoever. The connectome is fixed at what FlyWire measured. Every constant in `src/flynaf/config.py` was fitted against measurements of the simulated network rather than chosen by feel. -One thing is not a measurement. `arousal_multiplier` in `src/config.py` multiplies every weight in the matrix by 3, standing in for the neuromodulatory tone a brain in a body would have. The sign and the topology are untouched, the gain is uniform, and no edge is treated differently from any other, but the magnitude is not what FlyWire recorded. +One thing is not a measurement. `arousal_multiplier` in `src/flynaf/config.py` multiplies every weight in the matrix by 3, standing in for the neuromodulatory tone a brain in a body would have. The sign and the topology are untouched, the gain is uniform, and no edge is treated differently from any other, but the magnitude is not what FlyWire recorded. FlyWire maps the brain and stops at the neck. There is no ventral nerve cord here, so the descending neurons are read where they leave the brain rather than driving a simulated body. @@ -43,7 +45,7 @@ FlyWire maps the brain and stops at the neck. There is no ventral nerve cord her The simulation needs roughly 140 MB of FlyWire data that is deliberately not committed here. It comes from [eonsystemspbc/fly-brain](https://github.com/eonsystemspbc/fly-brain), which is where the neural engine this project builds on lives, and from the FlyWire annotation supplement. -`src/config.py` resolves the data path relative to the parent of this repository, so the layout on disk has to look like this. The folder this repository sits in can have any name. +`src/flynaf/config.py` resolves the data path relative to the parent of this repository, so the layout on disk has to look like this. The folder this repository sits in can have any name. ``` / @@ -85,7 +87,7 @@ For a developer the same thing is done from a terminal. git clone Fly-NAF cd Fly-NAF uv sync -uv run python src/scripts/fetch_data.py +uv run python -m flynaf.scripts.fetch_data ``` `fetch_data.py` prints the licences first, then fetches the two repositories and @@ -120,7 +122,7 @@ uv run python run.py ## Calibration The screen targets follow the game window. Once the game is the window in -front, its client rectangle is measured and every coordinate in `src/config.py` +front, its client rectangle is measured and every coordinate in `src/flynaf/config.py` is treated as a position inside a 1280x720 reference layout and scaled onto it. A window of a different size or in a different corner works without any measuring. The game draws at 1280x720 whatever the display, and in fullscreen @@ -131,13 +133,14 @@ A minimised window is never measured, because Windows reports it as a small rectangle far off screen. If the game is not in front when the night starts, the coordinates are used exactly as written, which assumes a 1280x720 window at the top left. The scripts below stay for that case, and for inspecting what -each region actually reads. +each region actually reads. Each one runs as a module, for example +`uv run python -m flynaf.scripts.select_roi`. -- `src/scripts/select_roi.py` takes a screenshot and lets you drag the two hallway capture boxes. -- `src/scripts/vision_calibrator.py` writes the measured values into `logs/`. -- `src/scripts/debug_vision.py` and `src/scripts/debug_camera.py` show live what each capture region is reading. -- `src/scripts/debug_reflex.py` shows the live contrast of both hallways against the threshold, and says whether a door would slam. Run it with a light held on, because a reading taken with the light off means nothing. -- `src/scripts/debug_tablet.py 1C` or `2A` records one camera with nothing in view and then prints the live drive it would send into the brain. That is how the spans in `TabletVision` are fitted, and how the CAM 2A button position, inferred from CAM 4B, is confirmed. +- `src/flynaf/scripts/select_roi.py` takes a screenshot and lets you drag the two hallway capture boxes. +- `src/flynaf/scripts/vision_calibrator.py` writes the measured values into `logs/`. +- `src/flynaf/scripts/debug_vision.py` and `src/flynaf/scripts/debug_camera.py` show live what each capture region is reading. +- `src/flynaf/scripts/debug_reflex.py` shows the live contrast of both hallways against the threshold, and says whether a door would slam. Run it with a light held on, because a reading taken with the light off means nothing. +- `src/flynaf/scripts/debug_tablet.py`, given `1C` or `2A`, records one camera with nothing in view and then prints the live drive it would send into the brain. That is how the spans in `TabletVision` are fitted, and how the CAM 2A button position, inferred from CAM 4B, is confirmed. The camera references are not cached. At the start of every night the fly raises the tablet once, records a few seconds of CAM 1C and CAM 2A while the curtain is closed and the hall is empty, and lowers it again, which also stalls Foxy while it happens. @@ -166,14 +169,15 @@ uv run python run.py ``` On Windows, double-clicking `Fly-NAF.bat` does the same. `run.py` is the only -entry point, `src/main.py` wires a night together and `src/night/` holds its parts (ADR 0024). +entry point, `src/flynaf/main.py` wires a night together and `src/flynaf/night/` holds its +parts (ADR 0024). A launcher window opens with the two choices the game's menu offers, New Game and Continue, and the night the save file is on. Picking one opens the game through Steam, loads the brain, brings the game to the front and clicks that option in the menu, so the night starts only once the brain is ready to play it. A copy outside Steam is opened through `GameLauncher.executable` in -`src/config.py`. `--new` and `--continue` do the same without the window. +`src/flynaf/config.py`. `--new` and `--continue` do the same without the window. The menu clicks need two points measured once, `new_game_x`, `new_game_y`, `continue_x` and `continue_y` in `GameLauncher`, in the same 1280x720 layout as @@ -206,21 +210,21 @@ A game that fills the screen gets the smaller overlay pinned over the office ins The claim at the top of this file is checkable without installing the game. ```bash -uv run python src/scripts/verify_connectome.py +uv run python -m flynaf.scripts.verify_connectome ``` It prints the neuron count, the edge count, the coverage against the FlyWire annotations, the super class of every neuron left out, the dimensions of the matrix handed to the engine, how many rows were dropped before it, and the gain in force. It needs the connectome data in place and nothing else. ```bash -uv run python src/scripts/measure_tablet_pathways.py -uv run python src/scripts/find_motor_pathway.py --source LC9 --target DNp09 +uv run python -m flynaf.scripts.measure_tablet_pathways +uv run python -m flynaf.scripts.find_motor_pathway --source LC9 --target DNp09 ``` The first drives each tablet channel into the running engine with the inhibitors on and prints which descending neuron answers and on which frame, the table ADR 0026 rests on. The second prints the cheapest excitatory path between any two FlyWire cell types. Every run also writes a trace to `logs/traces/`, one line per engine frame, carrying the input the eyes produced and what the descending neurons did with -it. `uv run python src/replay.py ` feeds that input back through a real +it. `uv run python -m flynaf.replay ` feeds that input back through a real engine and reports whether the giant fiber answers, how long it took and whether it stayed quiet on idle frames. @@ -232,18 +236,25 @@ recording off. ## Tests -There is no test framework dependency. Every file under `tests/` is a script -that runs its own cases and prints `ok` when they all hold. +The tests run under pytest, which `uv sync` installs with ruff as development +dependencies. ```bash -uv run python tests/test_hallway_detection.py +uv run pytest +uv run pytest -m "not connectome" +uv run ruff check src run.py tests ``` Most of them stub out the vision and motor layers and finish in under a second. `test_search_loop_wiring.py` plays scripted nights in real time and takes about -two minutes. Two of them, `test_inhibition.py` and `test_pathways.py`, run the -real engine on the full connectome, so they need the FlyWire data in place and -take several minutes each. +two minutes. Two of them, `test_inhibition.py` and `test_pathways.py`, carry the +`connectome` marker because they run the real engine on the full connectome. +They need the FlyWire data in place, take several minutes each, and are skipped +when the data is missing. The second command leaves them out. + +Every push to `main` and every pull request runs the lint and the tests on a +Windows runner. The FlyWire data cannot be redistributed, so the two connectome +tests are skipped there and run only on a machine that holds the data. ## Documentation @@ -258,17 +269,17 @@ and why the game is measured only once it is in front, and ADR 0022 covers the panel and the rule that only simulation variables drive it. ADR 0025 covers how populations are named by cell type rather than pasted as ids, and ADR 0026 and experiment 13 cover the tablet, Foxy and why DNp04 rather than the Giant Fiber -answers through it. +answers through it. ADR 0027 covers the `flynaf` package and the test pipeline. ## Licensing and credits This project is licensed under the GNU General Public License version 3 or any later version, in `LICENSE`. -It has to be. Two files under `src/neural/` are adapted from `code/run_pytorch.py` in [eonsystemspbc/fly-brain](https://github.com/eonsystemspbc/fly-brain). `models.py` carries the LIF neuron with alpha synapses, the delay buffer and the surrogate gradient, and `data_loader.py` carries the connectome loading. That repository is licensed under GPL version 2 or any later version, so this one inherits it, and version 3 is taken under the "or later" clause. The data files also come from its `data` folder. +It has to be. Two files under `src/flynaf/neural/` are adapted from `code/run_pytorch.py` in [eonsystemspbc/fly-brain](https://github.com/eonsystemspbc/fly-brain). `models.py` carries the LIF neuron with alpha synapses, the delay buffer and the surrogate gradient, and `data_loader.py` carries the connectome loading. That repository is licensed under GPL version 2 or any later version, so this one inherits it, and version 3 is taken under the "or later" clause. The data files also come from its `data` folder. Everything else here is original. An audit of all twenty source files against the upstream project found no meaningful overlap outside those two. -The fly in the launcher is the NeuroMechFly body model from [NeLy-EPFL/flygym](https://github.com/NeLy-EPFL/flygym), built from a micro-CT scan of an adult female fly and licensed under Apache 2.0, which the GPL v3 can include. `src/scripts/build_fly_model.py` downloads its simplified meshes at a fixed commit, poses them in the neutral stance and writes `src/env/assets/neuromechfly.bin.gz`, and the Apache licence travels with it as `src/env/assets/LICENSE-neuromechfly.txt`. The brain outline in the panel comes from [navis-flybrains](https://github.com/navis-org/navis-flybrains), GPL v3, made from the FAFB tissue mask, and is downloaded rather than committed. `src/env/assets/CHANGES-neuromechfly.txt` states what was changed from the original meshes, as the Apache licence asks of a derived file. The model is Lobato-Rios et al., *NeuroMechFly, a neuromechanical model of adult Drosophila melanogaster*, Nature Methods 2022, and Wang-Chen et al., *NeuroMechFly v2*, Nature Methods 2024, and both ask to be cited by anyone using the model, which this note does on their behalf. +The fly in the launcher is the NeuroMechFly body model from [NeLy-EPFL/flygym](https://github.com/NeLy-EPFL/flygym), built from a micro-CT scan of an adult female fly and licensed under Apache 2.0, which the GPL v3 can include. `src/flynaf/scripts/build_fly_model.py` downloads its simplified meshes at a fixed commit, poses them in the neutral stance and writes `src/flynaf/env/assets/neuromechfly.bin.gz`, and the Apache licence travels with it as `src/flynaf/env/assets/LICENSE-neuromechfly.txt`. The brain outline in the panel comes from [navis-flybrains](https://github.com/navis-org/navis-flybrains), GPL v3, made from the FAFB tissue mask, and is downloaded rather than committed. `src/flynaf/env/assets/CHANGES-neuromechfly.txt` states what was changed from the original meshes, as the Apache licence asks of a derived file. The model is Lobato-Rios et al., *NeuroMechFly, a neuromechanical model of adult Drosophila melanogaster*, Nature Methods 2022, and Wang-Chen et al., *NeuroMechFly v2*, Nature Methods 2024, and both ask to be cited by anyone using the model, which this note does on their behalf. The connectome is FlyWire 783, from the [FlyWire consortium](https://flywire.ai/). None of that data is redistributed here. It is released under CC BY-NC 4.0 and stays subject to FlyWire's own terms and citation requirements, so anyone using this project has to obtain it from the sources listed above. @@ -293,3 +304,4 @@ Five Nights at Freddy's is by Scott Cawthon and is not affiliated with this proj | 1.14 | Named run.py as the only entry point | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-30 | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-30 | | 1.15 | Pointed to the night package that main now wires | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-30 | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-30 | | 1.16 | Described the tablet watch on Foxy, the cell type populations and the new scripts, and corrected which tests are slow | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-30 | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-30 | +| 1.17 | Moved the paths and commands to the flynaf package, described the pytest and CI pipeline and the connectome marker | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-10-01 | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-10-01 | diff --git a/docs/adr/0027-the-source-is-the-flynaf-package-and-pytest-runs-it.md b/docs/adr/0027-the-source-is-the-flynaf-package-and-pytest-runs-it.md new file mode 100644 index 0000000..256ad0c --- /dev/null +++ b/docs/adr/0027-the-source-is-the-flynaf-package-and-pytest-runs-it.md @@ -0,0 +1,65 @@ +# ADR 0027: The Source Is The flynaf Package And pytest Runs It + +## Status +Accepted. + +## Context +Every module under `src/` was imported by its bare name, `import config`, +`from night.engine import ConnectomeEngine`. Nothing installed `src/`, so each +entry point put it on `sys.path` by hand. That was `run.py`, `replay.py`, every +script under `src/scripts/` and every one of the thirty-one test files. The +names it exposed at the top level were generic, `config`, `env`, `main`, +`replay`, and any installed distribution with the same name would have been +imported in their place depending on the order of the path. + +The tests had no runner. Each file was a script whose `__main__` block called +its cases by name, and the list had drifted from the cases. `test_fetch_data.py` +had no `__main__` block at all, so running it as a script ran nothing, and +`test_motor_pan_settle.py` defined a case its block never called. Neither +failure was visible, because a script that runs no assertion prints the same +`ok`. pytest collected all 194 fast cases without any change to them, but it was +not a dependency, and neither was ruff, so nothing checked either on a push. + +## Decision +The source moves to `src/flynaf/` and every import names the package, +`from flynaf import config`. `pyproject.toml` declares a hatchling build with +`src/flynaf` as its only package, so `uv sync` installs the project in editable +mode and no file touches `sys.path`. `run.py` stays at the root as the only +entry point, and the scripts and the replay run as modules, +`uv run python -m flynaf.scripts.fetch_data`. + +pytest and ruff join the project as the `dev` dependency group. The two tests +that run the real engine on the FlyWire data, `test_inhibition.py` and +`test_pathways.py`, carry a `connectome` marker, and `tests/conftest.py` skips +that marker when the connectivity parquet is not in place. ruff runs the +pyflakes rules, the import sorter and the error subset of pycodestyle, which the +code passes as it stands. + +A GitHub Actions workflow runs the lint and the tests on every push to `main` +and on every pull request. It runs on Windows, the platform the game runs on and +the one where PyTorch on PyPI is the CPU build. On Linux the same package pulls +the CUDA libraries, several gigabytes the runner would download and never use. + +## Consequences +The FlyWire data cannot be redistributed, so the runner never holds it and the +two connectome tests are reported as skipped there. Everything they check is +still checked on a machine with the data, and the skip names the missing path. + +`PROJECT_ROOT` in `config.py` climbs one directory more, since the file sits one +level deeper. The data still resolves to the folder that holds the repository, +and `logs/` and `tuning.toml` stay at the repository root. + +`uv sync` installs pytest and ruff for anyone who runs the project, including +the one click setup, a few megabytes against the PyTorch download it already +makes. + +The `__main__` blocks in the test files stay and still work, since the package +is installed, but pytest is the runner the README and the workflow use. + +Earlier records name the old paths. ADR 0024 places the night under +`src/night/`, which is now `src/flynaf/night/`, and the experiments keep the +paths they were written with. + +| Version | Description | Author(s) | Date | Reviewer(s) | Review Date | +|---|---|---|---|---|---| +| 1.0 | Recorded the move to the flynaf package, the pytest and ruff pipeline and the connectome marker | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-10-01 | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-10-01 | diff --git a/pyproject.toml b/pyproject.toml index a539793..30065ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,3 +17,31 @@ dependencies = [ "pyvista>=0.49.0", "torch>=2.13.0", ] + +[dependency-groups] +dev = [ + "pytest>=9.1.1", + "ruff>=0.16.9", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/flynaf"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +markers = [ + "connectome: runs the real engine on the FlyWire data, skipped when the data is not in place", +] + +[tool.ruff] +src = ["src"] + +[tool.ruff.lint] +select = ["E4", "E7", "E9", "F", "I"] + +[tool.ruff.lint.isort] +known-first-party = ["flynaf"] diff --git a/run.py b/run.py index a350edd..40e9399 100644 --- a/run.py +++ b/run.py @@ -2,15 +2,10 @@ import glob import logging import os -import sys -HERE = os.path.dirname(os.path.abspath(__file__)) -sys.path.insert(0, os.path.join(HERE, 'src')) - -import config -from env import desktop -import launcher -from scripts import fetch_data +from flynaf import config, launcher +from flynaf.env import desktop +from flynaf.scripts import fetch_data _log = logging.getLogger('run') @@ -37,14 +32,13 @@ def _report_platform(): def cmd_verify() -> int: - from scripts import verify_connectome + from flynaf.scripts import verify_connectome verify_connectome.main() return 0 def cmd_replay(trace: str | None) -> int: - import replay as replay_module - + from flynaf import replay as replay_module path = trace or _latest_trace() if path is None: _log.error( @@ -69,8 +63,7 @@ def _choose(session, mode: str | None): def cmd_play(mode: str | None) -> int: - import main as main_module - + from flynaf import main as main_module if not fetch_data.ready(): fetch_data.print_status() if not fetch_data.fetch(): diff --git a/src/night/__init__.py b/src/flynaf/__init__.py similarity index 100% rename from src/night/__init__.py rename to src/flynaf/__init__.py diff --git a/src/brain_adapter.py b/src/flynaf/brain_adapter.py similarity index 93% rename from src/brain_adapter.py rename to src/flynaf/brain_adapter.py index ede073b..a553fc7 100644 --- a/src/brain_adapter.py +++ b/src/flynaf/brain_adapter.py @@ -1,10 +1,11 @@ import logging -import torch from typing import List -import config -from neural.models import FlyBrainModel -from neural.data_loader import get_hash_tables, load_connectome_weights +import torch + +from flynaf import config +from flynaf.neural.data_loader import get_hash_tables, load_connectome_weights +from flynaf.neural.models import FlyBrainModel _log = logging.getLogger(__name__) diff --git a/src/config.py b/src/flynaf/config.py similarity index 98% rename from src/config.py rename to src/flynaf/config.py index 3c37f37..0a826c7 100644 --- a/src/config.py +++ b/src/flynaf/config.py @@ -2,8 +2,7 @@ from dataclasses import dataclass, field from typing import List - -PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) DATA_DIR = os.path.join(os.path.dirname(PROJECT_ROOT), 'fly-brain', 'data') CONNECTIVITY_PARQUET = os.path.join(DATA_DIR, '2025_Connectivity_783.parquet') diff --git a/src/env/anchor.py b/src/flynaf/env/anchor.py similarity index 100% rename from src/env/anchor.py rename to src/flynaf/env/anchor.py diff --git a/src/env/assets/CHANGES-neuromechfly.txt b/src/flynaf/env/assets/CHANGES-neuromechfly.txt similarity index 94% rename from src/env/assets/CHANGES-neuromechfly.txt rename to src/flynaf/env/assets/CHANGES-neuromechfly.txt index edf1a16..1cda494 100644 --- a/src/env/assets/CHANGES-neuromechfly.txt +++ b/src/flynaf/env/assets/CHANGES-neuromechfly.txt @@ -8,7 +8,7 @@ Source: the STL meshes under src/flygym/assets/model/neuromechfly/meshes/simplified_max2000faces/ and the rigging and neutral pose YAML files under src/flygym/assets/model/neuromechfly/, fetched from flygym at commit -38c8ec61034cd59bc5ba0de20688d4a3c0000d60 by src/scripts/build_fly_model.py. +38c8ec61034cd59bc5ba0de20688d4a3c0000d60 by src/flynaf/scripts/build_fly_model.py. Changes made: - Each body segment's mesh was placed in world space using the rigging diff --git a/src/env/assets/LICENSE-neuromechfly.txt b/src/flynaf/env/assets/LICENSE-neuromechfly.txt similarity index 100% rename from src/env/assets/LICENSE-neuromechfly.txt rename to src/flynaf/env/assets/LICENSE-neuromechfly.txt diff --git a/src/env/assets/neuromechfly.bin.gz b/src/flynaf/env/assets/neuromechfly.bin.gz similarity index 100% rename from src/env/assets/neuromechfly.bin.gz rename to src/flynaf/env/assets/neuromechfly.bin.gz diff --git a/src/env/brain_view.html b/src/flynaf/env/brain_view.html similarity index 100% rename from src/env/brain_view.html rename to src/flynaf/env/brain_view.html diff --git a/src/env/brain_view.py b/src/flynaf/env/brain_view.py similarity index 99% rename from src/env/brain_view.py rename to src/flynaf/env/brain_view.py index 6e5b2cb..414550a 100644 --- a/src/env/brain_view.py +++ b/src/flynaf/env/brain_view.py @@ -9,7 +9,7 @@ import cv2 import numpy as np -import config +from flynaf import config REGION_OPTIC = 0 REGION_CENTRAL = 1 diff --git a/src/env/calibrator.py b/src/flynaf/env/calibrator.py similarity index 99% rename from src/env/calibrator.py rename to src/flynaf/env/calibrator.py index 72e8d71..f03fc00 100644 --- a/src/env/calibrator.py +++ b/src/flynaf/env/calibrator.py @@ -1,7 +1,8 @@ import ctypes import time + import mss -import sys + class POINT(ctypes.Structure): _fields_ = [("x", ctypes.c_long), ("y", ctypes.c_long)] diff --git a/src/env/cascade.py b/src/flynaf/env/cascade.py similarity index 100% rename from src/env/cascade.py rename to src/flynaf/env/cascade.py diff --git a/src/env/desktop/__init__.py b/src/flynaf/env/desktop/__init__.py similarity index 100% rename from src/env/desktop/__init__.py rename to src/flynaf/env/desktop/__init__.py diff --git a/src/env/desktop/headless.py b/src/flynaf/env/desktop/headless.py similarity index 100% rename from src/env/desktop/headless.py rename to src/flynaf/env/desktop/headless.py diff --git a/src/env/desktop/linux.py b/src/flynaf/env/desktop/linux.py similarity index 100% rename from src/env/desktop/linux.py rename to src/flynaf/env/desktop/linux.py diff --git a/src/env/desktop/windows.py b/src/flynaf/env/desktop/windows.py similarity index 100% rename from src/env/desktop/windows.py rename to src/flynaf/env/desktop/windows.py diff --git a/src/env/input_controller.py b/src/flynaf/env/input_controller.py similarity index 98% rename from src/env/input_controller.py rename to src/flynaf/env/input_controller.py index 7349367..ca57f92 100644 --- a/src/env/input_controller.py +++ b/src/flynaf/env/input_controller.py @@ -1,11 +1,10 @@ +import logging import queue import threading import time -import logging -import config -from env import anchor -from env import desktop +from flynaf import config +from flynaf.env import anchor, desktop _CMD_QUEUE: queue.Queue = queue.Queue() _FACING = {'side': 'centre'} diff --git a/src/env/launcher.html b/src/flynaf/env/launcher.html similarity index 100% rename from src/env/launcher.html rename to src/flynaf/env/launcher.html diff --git a/src/env/overlay.py b/src/flynaf/env/overlay.py similarity index 99% rename from src/env/overlay.py rename to src/flynaf/env/overlay.py index ede6f23..0060635 100644 --- a/src/env/overlay.py +++ b/src/flynaf/env/overlay.py @@ -1,8 +1,7 @@ import time -import config -from env import anchor -from env import desktop +from flynaf import config +from flynaf.env import anchor, desktop find_browser = desktop.find_browser screen_size = desktop.screen_size diff --git a/src/env/profiles.py b/src/flynaf/env/profiles.py similarity index 100% rename from src/env/profiles.py rename to src/flynaf/env/profiles.py diff --git a/src/env/tablet_feed.py b/src/flynaf/env/tablet_feed.py similarity index 98% rename from src/env/tablet_feed.py rename to src/flynaf/env/tablet_feed.py index 682aa0f..326ccf8 100644 --- a/src/env/tablet_feed.py +++ b/src/flynaf/env/tablet_feed.py @@ -7,8 +7,8 @@ import mss import numpy as np -import config -from env import anchor +from flynaf import config +from flynaf.env import anchor _log = logging.getLogger(__name__) diff --git a/src/env/vision.py b/src/flynaf/env/vision.py similarity index 99% rename from src/env/vision.py rename to src/flynaf/env/vision.py index d516cce..8c9c42d 100644 --- a/src/env/vision.py +++ b/src/flynaf/env/vision.py @@ -1,17 +1,15 @@ +import collections +import logging import os import threading import time -import collections +import cv2 import mss import numpy as np -import cv2 - -import logging -import config -from env import anchor -from env import desktop +from flynaf import config +from flynaf.env import anchor, desktop _log = logging.getLogger(__name__) diff --git a/src/graph_search.py b/src/flynaf/graph_search.py similarity index 97% rename from src/graph_search.py rename to src/flynaf/graph_search.py index 2b5c7b0..115f3eb 100644 --- a/src/graph_search.py +++ b/src/flynaf/graph_search.py @@ -1,7 +1,9 @@ import heapq -import pandas as pd from collections import defaultdict -from typing import List, Dict, Optional, Tuple +from typing import Dict, List, Optional, Tuple + +import pandas as pd + def load_excitatory_graph(conn_path: str) -> Dict[int, List[Tuple[float, int]]]: df = pd.read_parquet(conn_path, columns=[ diff --git a/src/launcher.py b/src/flynaf/launcher.py similarity index 98% rename from src/launcher.py rename to src/flynaf/launcher.py index 370619b..df2b705 100644 --- a/src/launcher.py +++ b/src/flynaf/launcher.py @@ -6,10 +6,10 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from urllib.parse import parse_qs, urlparse -import config -from env import anchor, desktop, profiles -from env.overlay import anchor_to_game, find_game_window, hold_foreground -from scripts import fetch_data +from flynaf import config +from flynaf.env import anchor, desktop, profiles +from flynaf.env.overlay import anchor_to_game, find_game_window, hold_foreground +from flynaf.scripts import fetch_data _log = logging.getLogger(__name__) diff --git a/src/main.py b/src/flynaf/main.py similarity index 75% rename from src/main.py rename to src/flynaf/main.py index 4636e58..f1ad06d 100644 --- a/src/main.py +++ b/src/flynaf/main.py @@ -4,22 +4,21 @@ import torch -import config -from env.brain_view import SpikeFeed -from env.cascade import CascadeTracer -from env.input_controller import FNAFController, start_worker -from env.overlay import anchor_to_game -from env.tablet_feed import TabletFeed -from env.vision import FNAFVision -from night.engine import ConnectomeEngine -from night.panel import brain_view_task -from night.state import SaccadeRequest, SensoryState -from night.tablet.senses import TabletSenses -from night.tablet.watch import TabletWatch, build_gaze -from night.tasks import engine_task, saccade_task, vision_task -from recorder import SessionRecorder -from telemetry import ConnectomeTelemetry -import tuning +from flynaf import config, tuning +from flynaf.env.brain_view import SpikeFeed +from flynaf.env.cascade import CascadeTracer +from flynaf.env.input_controller import FNAFController, start_worker +from flynaf.env.overlay import anchor_to_game +from flynaf.env.tablet_feed import TabletFeed +from flynaf.env.vision import FNAFVision +from flynaf.night.engine import ConnectomeEngine +from flynaf.night.panel import brain_view_task +from flynaf.night.state import SaccadeRequest, SensoryState +from flynaf.night.tablet.senses import TabletSenses +from flynaf.night.tablet.watch import TabletWatch, build_gaze +from flynaf.night.tasks import engine_task, saccade_task, vision_task +from flynaf.recorder import SessionRecorder +from flynaf.telemetry import ConnectomeTelemetry _log = logging.getLogger(__name__) diff --git a/src/neural/cell_types.py b/src/flynaf/neural/cell_types.py similarity index 96% rename from src/neural/cell_types.py rename to src/flynaf/neural/cell_types.py index c6d908a..67190ae 100644 --- a/src/neural/cell_types.py +++ b/src/flynaf/neural/cell_types.py @@ -3,7 +3,7 @@ import pandas as pd -import config +from flynaf import config _log = logging.getLogger(__name__) @@ -21,7 +21,7 @@ def load(cls, annotations: str = config.ANNOTATIONS_TSV, return cls(pd.read_parquet(cache)) if not os.path.isfile(annotations): raise FileNotFoundError( - 'the FlyWire annotation table is missing at %s, run src/scripts/fetch_data.py' + 'the FlyWire annotation table is missing at %s, run python -m flynaf.scripts.fetch_data' % annotations ) _log.info('indexing FlyWire cell types from %s', os.path.basename(annotations)) diff --git a/src/neural/data_loader.py b/src/flynaf/neural/data_loader.py similarity index 99% rename from src/neural/data_loader.py rename to src/flynaf/neural/data_loader.py index abfc18c..46329c0 100644 --- a/src/neural/data_loader.py +++ b/src/flynaf/neural/data_loader.py @@ -18,11 +18,12 @@ this program. If not, see . """ import logging -import pandas as pd import pickle -import torch from pathlib import Path +import pandas as pd +import torch + _log = logging.getLogger(__name__) diff --git a/src/neural/models.py b/src/flynaf/neural/models.py similarity index 99% rename from src/neural/models.py rename to src/flynaf/neural/models.py index 33cc70d..fddaf08 100644 --- a/src/neural/models.py +++ b/src/flynaf/neural/models.py @@ -17,11 +17,11 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . """ +import numpy as np import torch import torch.nn as nn -import numpy as np -import config +from flynaf import config _P = config.NEURAL_PARAMS diff --git a/src/night/tablet/__init__.py b/src/flynaf/night/__init__.py similarity index 100% rename from src/night/tablet/__init__.py rename to src/flynaf/night/__init__.py diff --git a/src/night/calibration.py b/src/flynaf/night/calibration.py similarity index 89% rename from src/night/calibration.py rename to src/flynaf/night/calibration.py index 6cdde05..230e93b 100644 --- a/src/night/calibration.py +++ b/src/flynaf/night/calibration.py @@ -1,12 +1,12 @@ import asyncio import logging -import config -from env.input_controller import FNAFController -from env.overlay import anchor_to_game, game_in_front, window_title -from env.vision import FNAFVision -from night.motor import await_motor -from night.tablet.references import capture_camera_references +from flynaf import config +from flynaf.env.input_controller import FNAFController +from flynaf.env.overlay import anchor_to_game, game_in_front, window_title +from flynaf.env.vision import FNAFVision +from flynaf.night.motor import await_motor +from flynaf.night.tablet.references import capture_camera_references _log = logging.getLogger(__name__) diff --git a/src/night/doors.py b/src/flynaf/night/doors.py similarity index 96% rename from src/night/doors.py rename to src/flynaf/night/doors.py index f140f22..f78150d 100644 --- a/src/night/doors.py +++ b/src/flynaf/night/doors.py @@ -1,7 +1,7 @@ import logging -import config -from night.state import MotorRefrac, SensoryState +from flynaf import config +from flynaf.night.state import MotorRefrac, SensoryState _log = logging.getLogger(__name__) diff --git a/src/night/engine.py b/src/flynaf/night/engine.py similarity index 92% rename from src/night/engine.py rename to src/flynaf/night/engine.py index 4c7b365..62a7b10 100644 --- a/src/night/engine.py +++ b/src/flynaf/night/engine.py @@ -1,10 +1,10 @@ import torch -import config -from brain_adapter import BrainAdapter -from neural.cell_types import CellTypeIndex -from night.neurons import build_neuron_map, stimulus_rates -from night.state import SensoryState +from flynaf import config +from flynaf.brain_adapter import BrainAdapter +from flynaf.neural.cell_types import CellTypeIndex +from flynaf.night.neurons import build_neuron_map, stimulus_rates +from flynaf.night.state import SensoryState class ConnectomeEngine: diff --git a/src/night/monitor.py b/src/flynaf/night/monitor.py similarity index 96% rename from src/night/monitor.py rename to src/flynaf/night/monitor.py index 613f24c..9c3485f 100644 --- a/src/night/monitor.py +++ b/src/flynaf/night/monitor.py @@ -2,12 +2,12 @@ import logging import time -import config -from env.input_controller import FNAFController -from env.vision import FNAFVision -from night.motor import await_motor -from night.state import SensoryState -from night.tablet.watch import IdleTablet +from flynaf import config +from flynaf.env.input_controller import FNAFController +from flynaf.env.vision import FNAFVision +from flynaf.night.motor import await_motor +from flynaf.night.state import SensoryState +from flynaf.night.tablet.watch import IdleTablet _log = logging.getLogger(__name__) diff --git a/src/night/motor.py b/src/flynaf/night/motor.py similarity index 100% rename from src/night/motor.py rename to src/flynaf/night/motor.py diff --git a/src/night/neurons.py b/src/flynaf/night/neurons.py similarity index 98% rename from src/night/neurons.py rename to src/flynaf/night/neurons.py index e2b1756..5e3474c 100644 --- a/src/night/neurons.py +++ b/src/flynaf/night/neurons.py @@ -1,6 +1,6 @@ from dataclasses import dataclass -import config +from flynaf import config SIDES = ('left', 'right') STIMULUS_ROLES = ('eye', 'figure', 'loom_size', 'loom_speed') diff --git a/src/night/panel.py b/src/flynaf/night/panel.py similarity index 97% rename from src/night/panel.py rename to src/flynaf/night/panel.py index 4957dc0..b5005e7 100644 --- a/src/night/panel.py +++ b/src/flynaf/night/panel.py @@ -4,10 +4,10 @@ import subprocess import threading -import config -from env import desktop, profiles -from env.brain_view import BrainViewServer, SpikeFeed -from env.overlay import ( +from flynaf import config +from flynaf.env import desktop, profiles +from flynaf.env.brain_view import BrainViewServer, SpikeFeed +from flynaf.env.overlay import ( anchor_to_game, beside_game_rect, capture_regions, diff --git a/src/night/state.py b/src/flynaf/night/state.py similarity index 100% rename from src/night/state.py rename to src/flynaf/night/state.py diff --git a/src/flynaf/night/tablet/__init__.py b/src/flynaf/night/tablet/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/night/tablet/gaze.py b/src/flynaf/night/tablet/gaze.py similarity index 100% rename from src/night/tablet/gaze.py rename to src/flynaf/night/tablet/gaze.py diff --git a/src/night/tablet/references.py b/src/flynaf/night/tablet/references.py similarity index 93% rename from src/night/tablet/references.py rename to src/flynaf/night/tablet/references.py index 5c9052f..c4fa925 100644 --- a/src/night/tablet/references.py +++ b/src/flynaf/night/tablet/references.py @@ -1,9 +1,9 @@ import asyncio import logging -import config -from night.monitor import lower_monitor -from night.motor import await_motor +from flynaf import config +from flynaf.night.monitor import lower_monitor +from flynaf.night.motor import await_motor _log = logging.getLogger(__name__) diff --git a/src/night/tablet/senses.py b/src/flynaf/night/tablet/senses.py similarity index 89% rename from src/night/tablet/senses.py rename to src/flynaf/night/tablet/senses.py index 5219a39..fc3f912 100644 --- a/src/night/tablet/senses.py +++ b/src/flynaf/night/tablet/senses.py @@ -1,6 +1,6 @@ -import config -from night.state import SensoryState -from night.tablet.transduction import LoomChannel, camera_drive, figure_level +from flynaf import config +from flynaf.night.state import SensoryState +from flynaf.night.tablet.transduction import LoomChannel, camera_drive, figure_level class TabletSenses: diff --git a/src/night/tablet/transduction.py b/src/flynaf/night/tablet/transduction.py similarity index 100% rename from src/night/tablet/transduction.py rename to src/flynaf/night/tablet/transduction.py diff --git a/src/night/tablet/watch.py b/src/flynaf/night/tablet/watch.py similarity index 94% rename from src/night/tablet/watch.py rename to src/flynaf/night/tablet/watch.py index 6349999..24eb5f8 100644 --- a/src/night/tablet/watch.py +++ b/src/flynaf/night/tablet/watch.py @@ -1,8 +1,8 @@ import logging -import config -from night.state import SensoryState -from night.tablet.gaze import TabletGaze +from flynaf import config +from flynaf.night.state import SensoryState +from flynaf.night.tablet.gaze import TabletGaze _log = logging.getLogger(__name__) diff --git a/src/night/tasks.py b/src/flynaf/night/tasks.py similarity index 93% rename from src/night/tasks.py rename to src/flynaf/night/tasks.py index 82a05c3..38ec833 100644 --- a/src/night/tasks.py +++ b/src/flynaf/night/tasks.py @@ -2,21 +2,21 @@ import logging import time -import config -from env.brain_view import SpikeFeed -from env.input_controller import FNAFController -from env.vision import FNAFVision -from night.calibration import calibrate -from night.engine import ConnectomeEngine -from night.monitor import MonitorControl -from night.motor import await_motor -from night.doors import DoorControl -from night.state import SaccadeRequest, SensoryState -from night.tablet.senses import BlindSenses -from night.tablet.watch import IdleTablet -from recorder import SessionRecorder -from search_drive import ExploreDrive, SearchDrive -from telemetry import ConnectomeTelemetry +from flynaf import config +from flynaf.env.brain_view import SpikeFeed +from flynaf.env.input_controller import FNAFController +from flynaf.env.vision import FNAFVision +from flynaf.night.calibration import calibrate +from flynaf.night.doors import DoorControl +from flynaf.night.engine import ConnectomeEngine +from flynaf.night.monitor import MonitorControl +from flynaf.night.motor import await_motor +from flynaf.night.state import SaccadeRequest, SensoryState +from flynaf.night.tablet.senses import BlindSenses +from flynaf.night.tablet.watch import IdleTablet +from flynaf.recorder import SessionRecorder +from flynaf.search_drive import ExploreDrive, SearchDrive +from flynaf.telemetry import ConnectomeTelemetry _log = logging.getLogger(__name__) diff --git a/src/recorder.py b/src/flynaf/recorder.py similarity index 99% rename from src/recorder.py rename to src/flynaf/recorder.py index de70819..e6fec99 100644 --- a/src/recorder.py +++ b/src/flynaf/recorder.py @@ -4,7 +4,7 @@ import threading from datetime import datetime -import config +from flynaf import config _log = logging.getLogger(__name__) diff --git a/src/replay.py b/src/flynaf/replay.py similarity index 96% rename from src/replay.py rename to src/flynaf/replay.py index 72474be..e781755 100644 --- a/src/replay.py +++ b/src/flynaf/replay.py @@ -3,12 +3,8 @@ import logging import os import statistics -import sys -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) - -import config -import recorder +from flynaf import config, recorder _log = logging.getLogger(__name__) @@ -38,9 +34,8 @@ def _reflex_latencies(driven: list) -> list: def replay(trace_path: str, device: str = 'cpu') -> dict: - from brain_adapter import BrainAdapter - from night.engine import ConnectomeEngine - from night.state import SensoryState + from flynaf.night.engine import ConnectomeEngine + from flynaf.night.state import SensoryState header, frames, _events = recorder.load_trace(trace_path) if not frames: diff --git a/src/scripts/build_fly_model.py b/src/flynaf/scripts/build_fly_model.py similarity index 97% rename from src/scripts/build_fly_model.py rename to src/flynaf/scripts/build_fly_model.py index cc31cbe..8e4410f 100644 --- a/src/scripts/build_fly_model.py +++ b/src/flynaf/scripts/build_fly_model.py @@ -2,14 +2,11 @@ import json import os import struct -import sys import urllib.request import numpy as np -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) - -import config +from flynaf import config REVISION = '38c8ec61034cd59bc5ba0de20688d4a3c0000d60' BASE = ( @@ -17,7 +14,7 @@ '/src/flygym/assets/model/neuromechfly/' ) LICENSE_URL = f'https://raw.githubusercontent.com/NeLy-EPFL/flygym/{REVISION}/LICENSE' -ASSETS = os.path.join(config.PROJECT_ROOT, 'src', 'env', 'assets') +ASSETS = os.path.join(config.PROJECT_ROOT, 'src', 'flynaf', 'env', 'assets') OUTPUT = os.path.join(ASSETS, 'neuromechfly.bin.gz') SCALE = 1000.0 diff --git a/src/scripts/debug_camera.py b/src/flynaf/scripts/debug_camera.py similarity index 93% rename from src/scripts/debug_camera.py rename to src/flynaf/scripts/debug_camera.py index 91d4247..aa6cae9 100644 --- a/src/scripts/debug_camera.py +++ b/src/flynaf/scripts/debug_camera.py @@ -1,9 +1,12 @@ -import sys, os, time -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) +import os +import time + +import cv2 import mss import numpy as np -import cv2 -import config + +from flynaf import config + def main(): cx = config.CAMERA_DETECTION.patch_x diff --git a/src/scripts/debug_reflex.py b/src/flynaf/scripts/debug_reflex.py similarity index 90% rename from src/scripts/debug_reflex.py rename to src/flynaf/scripts/debug_reflex.py index a32810a..96c6685 100644 --- a/src/scripts/debug_reflex.py +++ b/src/flynaf/scripts/debug_reflex.py @@ -8,18 +8,12 @@ read while its light is on, exactly as in a real night, so a reading taken with the light off means nothing. - python src/scripts/debug_reflex.py + python -m flynaf.scripts.debug_reflex """ -import os -import sys import time -sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) - -import numpy as np - -import config -from env.vision import FNAFVision +from flynaf import config +from flynaf.env.vision import FNAFVision def main(): diff --git a/src/scripts/debug_tablet.py b/src/flynaf/scripts/debug_tablet.py similarity index 84% rename from src/scripts/debug_tablet.py rename to src/flynaf/scripts/debug_tablet.py index 9adf696..2964e50 100644 --- a/src/scripts/debug_tablet.py +++ b/src/flynaf/scripts/debug_tablet.py @@ -1,15 +1,11 @@ import argparse -import os -import sys import time -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) - -import config -from env.overlay import anchor_to_game -from env.tablet_feed import TabletFeed -from night.state import SensoryState -from night.tablet.senses import TabletSenses +from flynaf import config +from flynaf.env.overlay import anchor_to_game +from flynaf.env.tablet_feed import TabletFeed +from flynaf.night.state import SensoryState +from flynaf.night.tablet.senses import TabletSenses def main() -> int: diff --git a/src/scripts/debug_vision.py b/src/flynaf/scripts/debug_vision.py similarity index 90% rename from src/scripts/debug_vision.py rename to src/flynaf/scripts/debug_vision.py index 646f246..4c7deb6 100644 --- a/src/scripts/debug_vision.py +++ b/src/flynaf/scripts/debug_vision.py @@ -1,11 +1,8 @@ -import sys -import os -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) - +import cv2 import mss import numpy as np -import cv2 -import config + +from flynaf import config def run_debug(): diff --git a/src/scripts/fetch_data.py b/src/flynaf/scripts/fetch_data.py similarity index 98% rename from src/scripts/fetch_data.py rename to src/flynaf/scripts/fetch_data.py index 51d8d71..c9b12f5 100644 --- a/src/scripts/fetch_data.py +++ b/src/flynaf/scripts/fetch_data.py @@ -4,13 +4,10 @@ import os import shutil import subprocess -import sys import urllib.request import zipfile -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) - -import config +from flynaf import config PARENT = os.path.dirname(config.PROJECT_ROOT) FLY_BRAIN = os.path.join(PARENT, 'fly-brain') diff --git a/src/scripts/find_motor_pathway.py b/src/flynaf/scripts/find_motor_pathway.py similarity index 87% rename from src/scripts/find_motor_pathway.py rename to src/flynaf/scripts/find_motor_pathway.py index 542df64..c39d9fc 100644 --- a/src/scripts/find_motor_pathway.py +++ b/src/flynaf/scripts/find_motor_pathway.py @@ -1,13 +1,9 @@ import argparse -import os -import sys -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) - -import config -from graph_search import dijkstra, load_excitatory_graph -from neural.cell_types import CellTypeIndex -from neural.data_loader import get_hash_tables +from flynaf import config +from flynaf.graph_search import dijkstra, load_excitatory_graph +from flynaf.neural.cell_types import CellTypeIndex +from flynaf.neural.data_loader import get_hash_tables def _indices(cell_index: CellTypeIndex, flyid2i: dict, cell_type: str, side: str) -> list: diff --git a/src/scripts/measure_tablet_pathways.py b/src/flynaf/scripts/measure_tablet_pathways.py similarity index 94% rename from src/scripts/measure_tablet_pathways.py rename to src/flynaf/scripts/measure_tablet_pathways.py index 34679c1..c6b38de 100644 --- a/src/scripts/measure_tablet_pathways.py +++ b/src/flynaf/scripts/measure_tablet_pathways.py @@ -1,13 +1,9 @@ import argparse -import os import statistics -import sys -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) - -import config -from night.engine import ConnectomeEngine -from night.state import SensoryState +from flynaf import config +from flynaf.night.engine import ConnectomeEngine +from flynaf.night.state import SensoryState READOUTS = ('giant_fiber_left', 'looming_escape_left', 'explore') LEVELS = (0.1, 0.25, 0.5, 1.0) diff --git a/src/scripts/mine_inhibitors.py b/src/flynaf/scripts/mine_inhibitors.py similarity index 88% rename from src/scripts/mine_inhibitors.py rename to src/flynaf/scripts/mine_inhibitors.py index 71f557f..eb18b9d 100644 --- a/src/scripts/mine_inhibitors.py +++ b/src/flynaf/scripts/mine_inhibitors.py @@ -1,11 +1,7 @@ -import sys -import os - -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) - import pandas as pd -import config -from neural.cell_types import CellTypeIndex + +from flynaf import config +from flynaf.neural.cell_types import CellTypeIndex def mine_top_50(): diff --git a/src/scripts/select_roi.py b/src/flynaf/scripts/select_roi.py similarity index 92% rename from src/scripts/select_roi.py rename to src/flynaf/scripts/select_roi.py index f988082..75ffe00 100644 --- a/src/scripts/select_roi.py +++ b/src/flynaf/scripts/select_roi.py @@ -1,11 +1,8 @@ -import sys -import os import time -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) +import cv2 import mss import numpy as np -import cv2 def run_roi_selector(): diff --git a/src/scripts/verify_connectome.py b/src/flynaf/scripts/verify_connectome.py similarity index 95% rename from src/scripts/verify_connectome.py rename to src/flynaf/scripts/verify_connectome.py index aee1078..449d410 100644 --- a/src/scripts/verify_connectome.py +++ b/src/flynaf/scripts/verify_connectome.py @@ -1,11 +1,9 @@ -import sys import os -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) - import pandas as pd -import config -from brain_adapter import BrainAdapter + +from flynaf import config +from flynaf.brain_adapter import BrainAdapter FLYWIRE_ANNOTATIONS = config.ANNOTATIONS_TSV diff --git a/src/scripts/vision_calibrator.py b/src/flynaf/scripts/vision_calibrator.py similarity index 96% rename from src/scripts/vision_calibrator.py rename to src/flynaf/scripts/vision_calibrator.py index ef46bbf..87813a8 100644 --- a/src/scripts/vision_calibrator.py +++ b/src/flynaf/scripts/vision_calibrator.py @@ -1,13 +1,13 @@ import ctypes +import os +import sys import time -import mss + import cv2 +import mss import numpy as np -import os -import sys -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) -import config +from flynaf import config _LOG_DIR = os.path.join(config.PROJECT_ROOT, 'logs') _CANNY_LOW = 50 diff --git a/src/search_drive.py b/src/flynaf/search_drive.py similarity index 99% rename from src/search_drive.py rename to src/flynaf/search_drive.py index 70377a7..5634464 100644 --- a/src/search_drive.py +++ b/src/flynaf/search_drive.py @@ -1,4 +1,4 @@ -import config +from flynaf import config class AdaptiveSignal: diff --git a/src/telemetry.py b/src/flynaf/telemetry.py similarity index 99% rename from src/telemetry.py rename to src/flynaf/telemetry.py index 5a3f522..a23df9a 100644 --- a/src/telemetry.py +++ b/src/flynaf/telemetry.py @@ -1,8 +1,8 @@ -import time import os +import time from datetime import datetime -import config +from flynaf import config REFLEX_FRAMES = 10 diff --git a/src/tuning.py b/src/flynaf/tuning.py similarity index 99% rename from src/tuning.py rename to src/flynaf/tuning.py index 3ebfd96..444f8d4 100644 --- a/src/tuning.py +++ b/src/flynaf/tuning.py @@ -3,7 +3,7 @@ import os import tomllib -import config +from flynaf import config _log = logging.getLogger(__name__) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..bbd4600 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,14 @@ +import os + +import pytest + +from flynaf import config + + +def pytest_collection_modifyitems(items): + if os.path.exists(config.CONNECTIVITY_PARQUET): + return + skip = pytest.mark.skip(reason=f'the FlyWire data is not at {config.DATA_DIR}') + for item in items: + if 'connectome' in item.keywords: + item.add_marker(skip) diff --git a/tests/test_brain_view_geometry.py b/tests/test_brain_view_geometry.py index 71b9345..dd57650 100644 --- a/tests/test_brain_view_geometry.py +++ b/tests/test_brain_view_geometry.py @@ -2,18 +2,15 @@ import json import os import struct -import sys import tempfile from unittest.mock import patch import numpy as np -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) +from flynaf import config +from flynaf.env import brain_view -import config -from env import brain_view - -ASSET = os.path.join(config.PROJECT_ROOT, 'src', 'env', 'assets', 'neuromechfly.bin.gz') +ASSET = os.path.join(config.PROJECT_ROOT, 'src', 'flynaf', 'env', 'assets', 'neuromechfly.bin.gz') def _codex_file(folder: str, rows: list): diff --git a/tests/test_camera_reference_refresh.py b/tests/test_camera_reference_refresh.py index 473c97c..5d1f231 100644 --- a/tests/test_camera_reference_refresh.py +++ b/tests/test_camera_reference_refresh.py @@ -1,11 +1,7 @@ import asyncio -import sys -import os from unittest.mock import MagicMock -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) - -from night.monitor import reanchor_office_reference +from flynaf.night.monitor import reanchor_office_reference def _office_visible(): diff --git a/tests/test_cascade.py b/tests/test_cascade.py index 598a946..e1d8c9a 100644 --- a/tests/test_cascade.py +++ b/tests/test_cascade.py @@ -1,11 +1,7 @@ -import os -import sys import numpy as np -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) - -from env.cascade import CascadeTracer +from flynaf.env.cascade import CascadeTracer def _tracer(synapses: list, size: int, frames: list, **limits) -> CascadeTracer: diff --git a/tests/test_cell_types.py b/tests/test_cell_types.py index c135647..315a5ab 100644 --- a/tests/test_cell_types.py +++ b/tests/test_cell_types.py @@ -1,15 +1,12 @@ import os -import sys import tempfile -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) - import pandas as pd -import config -from neural.cell_types import CellTypeIndex -from night.neurons import NeuronMap, build_neuron_map, stimulus_rates -from night.state import SensoryState +from flynaf import config +from flynaf.neural.cell_types import CellTypeIndex +from flynaf.night.neurons import NeuronMap, build_neuron_map, stimulus_rates +from flynaf.night.state import SensoryState TABLE = pd.DataFrame({ 'root_id': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], diff --git a/tests/test_corollary_discharge.py b/tests/test_corollary_discharge.py index 76b6d05..65c4af6 100644 --- a/tests/test_corollary_discharge.py +++ b/tests/test_corollary_discharge.py @@ -1,12 +1,8 @@ import asyncio -import os -import sys -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) - -import config -from night import tasks -from night.state import SensoryState +from flynaf import config +from flynaf.night import tasks +from flynaf.night.state import SensoryState FULL_RATE = config.SIMULATION_PARAMS.base_sensory_rate_hz diff --git a/tests/test_door_idempotency.py b/tests/test_door_idempotency.py index d2bf719..5ece64c 100644 --- a/tests/test_door_idempotency.py +++ b/tests/test_door_idempotency.py @@ -1,13 +1,9 @@ -import os -import sys import queue import threading from unittest.mock import MagicMock, patch -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) - -import config -import env.input_controller as input_controller +import flynaf.env.input_controller as input_controller +from flynaf import config def _start_isolated_worker() -> queue.Queue: diff --git a/tests/test_door_release.py b/tests/test_door_release.py index 8d5ea4a..ed74d4f 100644 --- a/tests/test_door_release.py +++ b/tests/test_door_release.py @@ -1,9 +1,4 @@ -import os -import sys - -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) - -import config +from flynaf import config def decay_frames_to_release() -> int: diff --git a/tests/test_engine_rate_reset.py b/tests/test_engine_rate_reset.py index fa9c155..aa30f42 100644 --- a/tests/test_engine_rate_reset.py +++ b/tests/test_engine_rate_reset.py @@ -1,15 +1,11 @@ -import os -import sys from unittest.mock import MagicMock import torch -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) - -import config -from night.engine import ConnectomeEngine -from night.neurons import NeuronMap -from night.state import SensoryState +from flynaf import config +from flynaf.night.engine import ConnectomeEngine +from flynaf.night.neurons import NeuronMap +from flynaf.night.state import SensoryState NEURONS = 12 TOUCHED = set(range(8)) diff --git a/tests/test_explore_drive.py b/tests/test_explore_drive.py index a4f2526..dbeb2e6 100644 --- a/tests/test_explore_drive.py +++ b/tests/test_explore_drive.py @@ -1,10 +1,5 @@ -import os -import sys - -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) - -import config -from search_drive import ExploreDrive +from flynaf import config +from flynaf.search_drive import ExploreDrive DT = 1.0 / config.SIMULATION_PARAMS.target_fps P = config.EXPLORE_DYNAMICS diff --git a/tests/test_fetch_data.py b/tests/test_fetch_data.py index 3bc1d5f..6d79c3b 100644 --- a/tests/test_fetch_data.py +++ b/tests/test_fetch_data.py @@ -1,11 +1,8 @@ import os -import sys import zipfile from unittest.mock import patch -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) - -from scripts import fetch_data +from flynaf.scripts import fetch_data URL = 'https://github.com/example/repo' diff --git a/tests/test_focus_handback.py b/tests/test_focus_handback.py index 841f682..3f32060 100644 --- a/tests/test_focus_handback.py +++ b/tests/test_focus_handback.py @@ -1,10 +1,6 @@ -import os -import sys from unittest.mock import patch -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) - -from night import panel +from flynaf.night import panel GAME = 4242 PANEL = 99 diff --git a/tests/test_hallway_detection.py b/tests/test_hallway_detection.py index 53b4448..f7f5c1d 100644 --- a/tests/test_hallway_detection.py +++ b/tests/test_hallway_detection.py @@ -1,14 +1,10 @@ import collections -import os -import sys import threading import numpy as np -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) - -import config -from env.vision import FNAFVision +from flynaf import config +from flynaf.env.vision import FNAFVision SIZE = 32 diff --git a/tests/test_inhibition.py b/tests/test_inhibition.py index 9970d33..c59140a 100644 --- a/tests/test_inhibition.py +++ b/tests/test_inhibition.py @@ -1,11 +1,11 @@ -import sys -import os -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) +import pytest -import config -from night.engine import ConnectomeEngine -from night.state import SensoryState +from flynaf import config +from flynaf.night.engine import ConnectomeEngine +from flynaf.night.state import SensoryState + +pytestmark = pytest.mark.connectome FRAMES = 2500 TABLET_FRAMES = 40 diff --git a/tests/test_inspection_window.py b/tests/test_inspection_window.py index 8047c0a..07cdb72 100644 --- a/tests/test_inspection_window.py +++ b/tests/test_inspection_window.py @@ -1,12 +1,8 @@ import asyncio -import os -import sys -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) - -import config -from night import tasks -from night.state import SensoryState +from flynaf import config +from flynaf.night import tasks +from flynaf.night.state import SensoryState FRAMES = config.FORAGING_PARAMS.light_inspection_frames diff --git a/tests/test_launcher.py b/tests/test_launcher.py index 1066bdb..6a6e72d 100644 --- a/tests/test_launcher.py +++ b/tests/test_launcher.py @@ -1,13 +1,9 @@ import dataclasses import os -import sys import tempfile from unittest.mock import patch -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) - -import config -import launcher +from flynaf import config, launcher GAME = 919200 CALIBRATED = dataclasses.replace( diff --git a/tests/test_monitor_lowering.py b/tests/test_monitor_lowering.py index 7ac4c59..64c6ff8 100644 --- a/tests/test_monitor_lowering.py +++ b/tests/test_monitor_lowering.py @@ -1,12 +1,9 @@ import asyncio -import os -import sys -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) - -from night import monitor -from night.state import SensoryState -from telemetry import ConnectomeTelemetry +from flynaf import config +from flynaf.night import monitor +from flynaf.night.state import SensoryState +from flynaf.telemetry import ConnectomeTelemetry class _ImmediateEvent: @@ -14,8 +11,6 @@ def wait(self, timeout=None): return True -import config - CONFIRM = config.CAMERA_DETECTION.lower_confirm_sec BUDGET = config.CAMERA_DETECTION.lower_gesture_attempts diff --git a/tests/test_motor_pan_settle.py b/tests/test_motor_pan_settle.py index 2feb679..53281cc 100644 --- a/tests/test_motor_pan_settle.py +++ b/tests/test_motor_pan_settle.py @@ -1,13 +1,9 @@ -import os -import sys import queue import threading from unittest.mock import MagicMock, patch -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) - -import config -import env.input_controller as input_controller +import flynaf.env.input_controller as input_controller +from flynaf import config def _start_isolated_worker() -> queue.Queue: diff --git a/tests/test_motor_suppression.py b/tests/test_motor_suppression.py index a6ee207..b9ff1b7 100644 --- a/tests/test_motor_suppression.py +++ b/tests/test_motor_suppression.py @@ -1,12 +1,5 @@ import unittest -from unittest.mock import MagicMock -import asyncio -import sys -import os -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) - -from night.state import SensoryState def evaluate_motor_action(spiked: bool, camera_open: bool) -> bool: if spiked and not camera_open: diff --git a/tests/test_overlay_placement.py b/tests/test_overlay_placement.py index 7f43c97..3662fe5 100644 --- a/tests/test_overlay_placement.py +++ b/tests/test_overlay_placement.py @@ -1,13 +1,8 @@ -import os -import sys - -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) - from unittest.mock import patch -import config -from env import anchor -from env.overlay import ( +from flynaf import config +from flynaf.env import anchor +from flynaf.env.overlay import ( beside_game_rect, capture_regions, centered_rect, diff --git a/tests/test_pathways.py b/tests/test_pathways.py index 10938b2..ea24f75 100644 --- a/tests/test_pathways.py +++ b/tests/test_pathways.py @@ -1,11 +1,11 @@ -import sys -import os - -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) +import pytest import torch -from night.engine import ConnectomeEngine -from night.state import SensoryState + +from flynaf.night.engine import ConnectomeEngine +from flynaf.night.state import SensoryState + +pytestmark = pytest.mark.connectome FRAMES = 2500 diff --git a/tests/test_saccade_scheduling.py b/tests/test_saccade_scheduling.py index ba920da..6d6bb90 100644 --- a/tests/test_saccade_scheduling.py +++ b/tests/test_saccade_scheduling.py @@ -1,10 +1,5 @@ -import os -import sys - -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) - -import config -from search_drive import SearchDrive +from flynaf import config +from flynaf.search_drive import SearchDrive SPAN = 100 # 50 sensory neurons x 2 steps per frame DT = 1.0 / config.SIMULATION_PARAMS.target_fps @@ -129,7 +124,10 @@ def test_a_static_offset_is_adapted_away_instead_of_parking_the_fly(): drive = SearchDrive(SPAN) clock = Clock() drift = measured_drift(1500) - biased = lambda i: 25.0 + drift[i] + + def biased(i): + return 25.0 + drift[i] + looks = feed(drive, clock, 1500, membrane_diff=biased, busy_frames=BUSY_FRAMES) sides = [side for _, side, _ in looks] assert sides.count('left') > 0 and sides.count('right') > 0 diff --git a/tests/test_search_loop_wiring.py b/tests/test_search_loop_wiring.py index 3c823f6..8cd24cf 100644 --- a/tests/test_search_loop_wiring.py +++ b/tests/test_search_loop_wiring.py @@ -6,19 +6,13 @@ every time the fly looks at something. """ import asyncio -import os -import sys from unittest.mock import patch -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) - import torch -import config -from night import tasks -from night import calibration -from night.state import SaccadeRequest, SensoryState -from telemetry import ConnectomeTelemetry +from flynaf.night import calibration, tasks +from flynaf.night.state import SaccadeRequest, SensoryState +from flynaf.telemetry import ConnectomeTelemetry NEURONS = 8 TYPICAL = 30.0 diff --git a/tests/test_startup_readiness.py b/tests/test_startup_readiness.py index 7c5b0ec..c7be617 100644 --- a/tests/test_startup_readiness.py +++ b/tests/test_startup_readiness.py @@ -1,13 +1,9 @@ import asyncio -import os -import sys from unittest.mock import MagicMock, patch -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) - -import config -from env import overlay -from night import calibration +from flynaf import config +from flynaf.env import overlay +from flynaf.night import calibration GAME = 789498 EDITOR = 66778 diff --git a/tests/test_tablet_escape.py b/tests/test_tablet_escape.py index 9db506f..a219c0c 100644 --- a/tests/test_tablet_escape.py +++ b/tests/test_tablet_escape.py @@ -1,13 +1,9 @@ import asyncio -import os -import sys -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) - -from night import monitor, tasks -from night.doors import DoorControl -from night.state import SensoryState -from telemetry import ConnectomeTelemetry +from flynaf.night import monitor, tasks +from flynaf.night.doors import DoorControl +from flynaf.night.state import SensoryState +from flynaf.telemetry import ConnectomeTelemetry class _Done: diff --git a/tests/test_tablet_gaze.py b/tests/test_tablet_gaze.py index 3c55c18..e0935f5 100644 --- a/tests/test_tablet_gaze.py +++ b/tests/test_tablet_gaze.py @@ -1,13 +1,8 @@ -import os -import sys - -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) - -import config -from night.state import SensoryState -from night.tablet.gaze import TabletGaze -from night.tablet.watch import TabletWatch -from telemetry import ConnectomeTelemetry +from flynaf import config +from flynaf.night.state import SensoryState +from flynaf.night.tablet.gaze import TabletGaze +from flynaf.night.tablet.watch import TabletWatch +from flynaf.telemetry import ConnectomeTelemetry RAISE = 1.2 SWITCH = 0.5 diff --git a/tests/test_tablet_senses.py b/tests/test_tablet_senses.py index 44a74dc..df40934 100644 --- a/tests/test_tablet_senses.py +++ b/tests/test_tablet_senses.py @@ -1,10 +1,5 @@ -import os -import sys - -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) - -from night.state import SensoryState -from night.tablet.senses import TabletSenses +from flynaf.night.state import SensoryState +from flynaf.night.tablet.senses import TabletSenses class ScriptedFeed: diff --git a/tests/test_tablet_transduction.py b/tests/test_tablet_transduction.py index b3686ed..259e72f 100644 --- a/tests/test_tablet_transduction.py +++ b/tests/test_tablet_transduction.py @@ -1,12 +1,7 @@ -import os -import sys - -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) - import numpy as np -from env.tablet_feed import bank_distance, bank_noise, prepare, spread_sample -from night.tablet.transduction import LoomChannel, camera_drive, figure_level +from flynaf.env.tablet_feed import bank_distance, bank_noise, prepare, spread_sample +from flynaf.night.tablet.transduction import LoomChannel, camera_drive, figure_level FIGURE_CASES = ( (0.0, 100.0, 1000.0, 0.0), diff --git a/tests/test_trace_roundtrip.py b/tests/test_trace_roundtrip.py index 82f5f64..8b6d4bf 100644 --- a/tests/test_trace_roundtrip.py +++ b/tests/test_trace_roundtrip.py @@ -1,12 +1,8 @@ import os -import sys import tempfile -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) - -import recorder -import replay -from night.state import SensoryState +from flynaf import recorder, replay +from flynaf.night.state import SensoryState def FakeState(left, right, inhib, **tablet): diff --git a/tests/test_tuning_bounds.py b/tests/test_tuning_bounds.py index 7c57f23..538bc1b 100644 --- a/tests/test_tuning_bounds.py +++ b/tests/test_tuning_bounds.py @@ -1,11 +1,7 @@ import os -import sys import tempfile -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) - -import config -import tuning +from flynaf import config, tuning def test_every_knob_points_at_a_real_config_field(): diff --git a/tests/test_vision_buffers.py b/tests/test_vision_buffers.py index 05b17ea..43e3718 100644 --- a/tests/test_vision_buffers.py +++ b/tests/test_vision_buffers.py @@ -1,14 +1,11 @@ import collections import os -import sys import threading import time import numpy as np -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) - -from env.vision import FNAFVision +from flynaf.env.vision import FNAFVision def make_vision(calib_frames=5, calib_delay=0.01) -> FNAFVision: diff --git a/tests/test_vision_reference_cache.py b/tests/test_vision_reference_cache.py index f72407d..3fabf85 100644 --- a/tests/test_vision_reference_cache.py +++ b/tests/test_vision_reference_cache.py @@ -1,12 +1,8 @@ -import os -import sys import tempfile import numpy as np -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) - -from env.vision import FNAFVision +from flynaf.env.vision import FNAFVision def make_vision(reference_dir: str, l_bbox: int = 4, r_bbox: int = 6) -> FNAFVision: diff --git a/uv.lock b/uv.lock index e70425e..9d8c0d8 100644 --- a/uv.lock +++ b/uv.lock @@ -530,7 +530,7 @@ wheels = [ [[package]] name = "fly-naf" version = "0.1.0" -source = { virtual = "." } +source = { editable = "." } dependencies = [ { name = "caveclient" }, { name = "mss" }, @@ -545,6 +545,12 @@ dependencies = [ { name = "torch" }, ] +[package.dev-dependencies] +dev = [ + { name = "pytest" }, + { name = "ruff" }, +] + [package.metadata] requires-dist = [ { name = "caveclient", specifier = ">=5.21.0" }, @@ -559,6 +565,12 @@ requires-dist = [ { name = "torch", specifier = ">=2.13.0" }, ] +[package.metadata.requires-dev] +dev = [ + { name = "pytest", specifier = ">=9.1.1" }, + { name = "ruff", specifier = ">=0.16.9" }, +] + [[package]] name = "fonttools" version = "4.65.0" @@ -642,6 +654,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/57/b0/0e52c878c53f245edd3a11020f20979b3f490f245af532c7cae3027754b5/idna-3.19-py3-none-any.whl", hash = "sha256:815e7be7a7806d54abb586dc943addc79e8b2ee16915059658cbeff4b1b43bf4", size = 68550, upload-time = "2026-08-18T05:14:22.343Z" }, ] +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + [[package]] name = "ipython" version = "9.17.1" @@ -1580,6 +1601,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f4/e1/5b7b8bbb55084d1425bcb9bc823ff519e1b2be05f6ebb0089e2eacc38413/platformdirs-4.11.8-py3-none-any.whl", hash = "sha256:52f2f181bbfde907966932cc8312d967d02976422d66d537ea16092b8e291081", size = 24027, upload-time = "2026-09-08T22:20:41.537Z" }, ] +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + [[package]] name = "pooch" version = "1.9.0" @@ -1748,6 +1778,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl", hash = "sha256:850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d", size = 122781, upload-time = "2026-01-21T03:57:55.912Z" }, ] +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -2002,6 +2048,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/10/85/48f0abdcef5cce4e034c7a5b0ceeceba0b01bf0d942824f4bb720afe2dec/rpds_py-2026.6.3-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:8e65860d238379ed982fd9ba690579b5e95af2f4840f99c772816dbe573cb826", size = 586486, upload-time = "2026-06-30T07:17:51.141Z" }, ] +[[package]] +name = "ruff" +version = "0.16.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/96/bf/c935ca98e73fe8ce65b87ef08a280c0c1e85295d569228c15e87d8fdfaf1/ruff-0.16.9.tar.gz", hash = "sha256:12b625c6cfba78d285d9f48eda5f053374f1e53cb10ef17342a383750db99161", size = 4948764, upload-time = "2026-09-24T20:37:49.416Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0d/26/df51322b52ee1ada7eff2d071ea09d11d5c2d1dcc9f02594f5785c4d1635/ruff-0.16.9-py3-none-linux_armv6l.whl", hash = "sha256:95e6f022090368ab3b824c36276839c53b2adf1a3f4c09fefc33dfc400f6da96", size = 10082922, upload-time = "2026-09-24T20:37:13.045Z" }, + { url = "https://files.pythonhosted.org/packages/a5/27/7bf51f5a7aa375e9f339280a303aab44ca75dc1525f1cdc5991761685b0f/ruff-0.16.9-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:a5f27be168556594a86d2f415db0cf43f5291917849318f873c7e2791f7a8c67", size = 10236360, upload-time = "2026-09-24T20:37:16.049Z" }, + { url = "https://files.pythonhosted.org/packages/b6/63/09659283f92f02dff45809da194a70da2f688d87c55d8875c4fae3536072/ruff-0.16.9-py3-none-macosx_11_0_arm64.whl", hash = "sha256:1632eb1d6197f33bd00b1acbc5b71009e89a8895c158e2d2b03a834fac964ab6", size = 9892940, upload-time = "2026-09-24T20:37:17.957Z" }, + { url = "https://files.pythonhosted.org/packages/24/58/98de1b72ec172f5f8f1731236fe21585b3998bf7dfe9fcc44ae9ba626012/ruff-0.16.9-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b3f951b14d865d5952c89d40a5ca07e87abe24fa5453299878411e127748fb1c", size = 10032114, upload-time = "2026-09-24T20:37:19.942Z" }, + { url = "https://files.pythonhosted.org/packages/c8/7d/f1e17c54ab59d4bad1dce8ee3e22a7a1d0ef4745240decacdcf3832b5bb2/ruff-0.16.9-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:447fc07e1573afff7cb02803462b12b6c8ece7cf10e2cd78565fa6d7a1c0bf8d", size = 9910227, upload-time = "2026-09-24T20:37:21.872Z" }, + { url = "https://files.pythonhosted.org/packages/34/19/436f647a65075bbd3bab2668b3bdaa5120559b294694018cdcefabbbf30b/ruff-0.16.9-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8a3e039a6a40ed976c491722b60e0ae4a4aa1a86057f540ee7a37a5d19ae9120", size = 10547484, upload-time = "2026-09-24T20:37:24.229Z" }, + { url = "https://files.pythonhosted.org/packages/03/59/38430a6bc2f6d8095447ac39625cf8b6e9344a47e6c26225c8ba1bff3ffb/ruff-0.16.9-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4684dded7db60aa57cb118fa158630f5feade4af5782903b6053484bdf9bd129", size = 11412367, upload-time = "2026-09-24T20:37:26.307Z" }, + { url = "https://files.pythonhosted.org/packages/c8/bd/bbb6d7fc7f208c8b8c50dd5dc8206e4cfdb1e7a8fb852606a3adf370c880/ruff-0.16.9-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d29c934357e45642fda2f34c0b1f4025b4a6c01e15e4bf0016879d60078a142c", size = 10869787, upload-time = "2026-09-24T20:37:28.35Z" }, + { url = "https://files.pythonhosted.org/packages/bc/b8/9c543074918061abbefc3bd139bee22de35abedb00dde0f2d27288838962/ruff-0.16.9-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a21713e629d3e5bdb2f5c2def1cc7f04f47fa8e1a7eb0571b4a28e1da64bc728", size = 10406494, upload-time = "2026-09-24T20:37:30.624Z" }, + { url = "https://files.pythonhosted.org/packages/35/7a/5a8851bd146e7ccf8fd4b003f6c75c11f8fbdb0e60673b45097986b6bf41/ruff-0.16.9-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:7baa24ef5fc8e77aa93879e1d3f43754a01ae488e869f1ae30cf431afd4d2452", size = 10590083, upload-time = "2026-09-24T20:37:32.439Z" }, + { url = "https://files.pythonhosted.org/packages/87/f0/4c3467188f23f806960b46fa76575a7cd0514c9ba90562650b71efc96980/ruff-0.16.9-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:a41aac6230aadfaa133bdfa1614488531ffa3e0837567ae04c0da2058a9c0f9e", size = 10119151, upload-time = "2026-09-24T20:37:34.581Z" }, + { url = "https://files.pythonhosted.org/packages/15/34/5a4def5adea572ce6aea0bb64f21f928ee317b80e0db747d7979b01d7261/ruff-0.16.9-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:c2529fb5896d49115b0e9aa8f887490b34bbe76baf879ec2264ac59406869ce7", size = 9911544, upload-time = "2026-09-24T20:37:36.796Z" }, + { url = "https://files.pythonhosted.org/packages/62/5d/d15ebea7499eef9373318c0ee6ca127832927c6529731f6d48e18dce7ca9/ruff-0.16.9-py3-none-musllinux_1_2_i686.whl", hash = "sha256:41e3870277694177429b56406d65dfbdb2c2802c52b715edaf6a0b829c69d4ee", size = 10269884, upload-time = "2026-09-24T20:37:38.857Z" }, + { url = "https://files.pythonhosted.org/packages/d1/56/c5d3cd119ded7a3c7aba0e961b69cb3df701c91662c98ad694467d060ce1/ruff-0.16.9-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:8adbe4e58af167f767d7b2ba5e83c42e878350796cf78c2f5e14ab9903a92588", size = 10749366, upload-time = "2026-09-24T20:37:41.042Z" }, + { url = "https://files.pythonhosted.org/packages/ac/fe/734ec7527029ac757ecf821f143c9f3fcf69149c21f53a044a899b430f5a/ruff-0.16.9-py3-none-win32.whl", hash = "sha256:0e1dbc2073624dee6618d41d0098690a7244654af746704b64759e12b6b6b385", size = 10152355, upload-time = "2026-09-24T20:37:43.025Z" }, + { url = "https://files.pythonhosted.org/packages/14/21/26e4643629b3ebb44f0a06f9c9a53058d63d989415f63a9a3c28e2ee7f22/ruff-0.16.9-py3-none-win_amd64.whl", hash = "sha256:6bd40fec8cd4c8a3d4dd589bd8ad4e6320c13c29234159bfd959a40d529d597b", size = 10592965, upload-time = "2026-09-24T20:37:44.944Z" }, + { url = "https://files.pythonhosted.org/packages/51/60/5fb1a39dbb5ae314d5f59bc7348a63c1d5c20f3cd83914c4b5cb0be31d2d/ruff-0.16.9-py3-none-win_arm64.whl", hash = "sha256:ed1a252039200f57a59eebc063b54beabea67bfbaaca0eeaa7f54b5fbcda2284", size = 10458649, upload-time = "2026-09-24T20:37:46.882Z" }, +] + [[package]] name = "scooby" version = "0.12.0" From c5b2affb5e8da807f0792525c27b8fcef695815a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Artur=20Mendon=C3=A7a=20Arruda?= Date: Thu, 1 Oct 2026 20:07:36 -0300 Subject: [PATCH 09/14] feat(panel): cut the browser title bar off the panel and round its corners --- src/flynaf/config.py | 1 + src/flynaf/env/brain_view.html | 78 ++++++++++++++++++++++++------ src/flynaf/env/brain_view.py | 16 ++++++ src/flynaf/env/desktop/__init__.py | 8 +++ src/flynaf/env/desktop/headless.py | 8 +++ src/flynaf/env/desktop/linux.py | 8 +++ src/flynaf/env/desktop/windows.py | 18 +++++++ src/flynaf/env/overlay.py | 41 +++++++++++++++- src/flynaf/night/panel.py | 20 ++++++-- tests/test_browser_chrome.py | 35 ++++++++++++++ 10 files changed, 211 insertions(+), 22 deletions(-) create mode 100644 tests/test_browser_chrome.py diff --git a/src/flynaf/config.py b/src/flynaf/config.py index 0a826c7..09d9289 100644 --- a/src/flynaf/config.py +++ b/src/flynaf/config.py @@ -119,6 +119,7 @@ class BrainView: patch_interval_sec: float = 0.3 patch_pixels: int = 72 beside_min_width: int = 420 + corner_radius: int = 14 launch_browser: bool = True diff --git a/src/flynaf/env/brain_view.html b/src/flynaf/env/brain_view.html index 8f196cd..7d2d3b1 100644 --- a/src/flynaf/env/brain_view.html +++ b/src/flynaf/env/brain_view.html @@ -8,8 +8,10 @@ :root { --bg: #07070a; --panel: #0c0c11; - --line: #1b1b24; - --label: #6a6d78; + --line: rgba(150, 160, 200, 0.09); + --edge: rgba(150, 170, 255, 0.16); + --well: #101017; + --label: #6f7280; --value: #e8e9ee; --optic: #4a9eff; --central: #ffc94a; @@ -17,6 +19,7 @@ --motor: #ff5c4a; --tablet: #a855f7; --light: #f5b94a; + --corner: 0px; } * { margin: 0; padding: 0; box-sizing: border-box; } html, body { height: 100%; background: var(--bg); overflow: hidden; } @@ -26,16 +29,33 @@ display: flex; flex-direction: column; } + body::after { + content: ''; position: fixed; inset: 2px; z-index: 10; pointer-events: none; + border: 1px solid var(--edge); + border-radius: max(0px, calc(var(--corner) - 2px)); + } header { padding: 10px 14px 9px; border-bottom: 1px solid var(--line); + background: linear-gradient(180deg, rgba(150, 170, 255, 0.05), rgba(150, 170, 255, 0)); display: flex; justify-content: space-between; align-items: baseline; gap: 10px; } h1 { font-size: 10.5px; letter-spacing: 0.14em; font-weight: 600; } - #state { font-size: 9.5px; letter-spacing: 0.12em; color: var(--label); white-space: nowrap; } + h1::before { + content: ''; display: inline-block; width: 6px; height: 6px; margin: 0 8px 1px 0; + border-radius: 50%; background: var(--sensory); box-shadow: 0 0 8px var(--sensory); + animation: live 2.4s ease-in-out infinite; + } + body.offline h1::before { background: var(--motor); box-shadow: 0 0 8px var(--motor); animation: none; } + @keyframes live { 50% { opacity: 0.35; } } + #state { + font-size: 9.5px; letter-spacing: 0.12em; color: var(--label); white-space: nowrap; + padding: 2px 8px; margin: -3px 0; border-radius: 999px; + border: 1px solid var(--line); background: rgba(255, 255, 255, 0.025); + } #stimulus { display: grid; @@ -54,8 +74,11 @@ position: relative; width: 58px; height: 58px; flex: none; - background: #111117; - outline: 1px solid var(--line); + background: var(--well); + border-radius: 7px; + overflow: hidden; + box-shadow: 0 0 0 1px var(--line); + transition: box-shadow 200ms ease; } .thumb img { width: 100%; height: 100%; display: block; image-rendering: pixelated; } .thumb b { @@ -65,13 +88,13 @@ color: var(--label); background: rgba(7, 7, 10, 0.72); } .eye.look .thumb b { display: none; } - .eye.look .thumb { outline-color: var(--light); } - .eye.on .thumb { outline: 2px solid var(--sensory); box-shadow: 0 0 12px rgba(74, 222, 128, 0.45); } + .eye.look .thumb { box-shadow: 0 0 0 1px var(--light), 0 0 10px rgba(245, 185, 74, 0.18); } + .eye.on .thumb { box-shadow: 0 0 0 2px var(--sensory), 0 0 14px rgba(74, 222, 128, 0.45); } .info { flex: 1; min-width: 0; font-size: 9px; letter-spacing: 0.06em; color: var(--label); } .info .name { color: var(--value); letter-spacing: 0.1em; margin-bottom: 5px; } - .meter { position: relative; height: 5px; background: #14141b; margin: 4px 0 3px; } - .meter i { position: absolute; left: 0; top: 0; bottom: 0; width: 0; background: var(--light); transition: width 90ms linear; } - .meter u { position: absolute; left: 50%; top: -2px; bottom: -2px; width: 1px; background: var(--value); } + .meter { position: relative; height: 5px; background: var(--well); border-radius: 3px; margin: 4px 0 3px; } + .meter i { position: absolute; left: 0; top: 0; bottom: 0; width: 0; border-radius: 3px; background: var(--light); transition: width 90ms linear; } + .meter u { position: absolute; left: 50%; top: -2px; bottom: -2px; width: 1px; border-radius: 1px; background: var(--value); opacity: 0.8; } .eye.on .meter i { background: var(--sensory); } .pill { margin-top: 4px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; } .eye.on .pill { color: var(--sensory); } @@ -103,17 +126,21 @@ #caption em { font-style: normal; color: var(--value); } #caption .src { color: var(--sensory); } #caption .replay { - display: inline-block; margin-top: 4px; padding: 1px 6px; - border: 1px solid var(--line); color: var(--label); font-size: 8px; letter-spacing: 0.12em; + display: inline-block; margin-top: 4px; padding: 1px 8px; border-radius: 999px; + border: 1px solid var(--line); background: rgba(7, 7, 10, 0.6); + color: var(--label); font-size: 8px; letter-spacing: 0.12em; } #timeline-wrap { border-top: 1px solid var(--line); position: relative; } #timeline { width: 100%; height: 150px; } - #counters { display: flex; border-top: 1px solid var(--line); } + #counters { + display: flex; border-top: 1px solid var(--line); + background: linear-gradient(0deg, rgba(150, 170, 255, 0.035), rgba(150, 170, 255, 0)); + } .counter { flex: 1; padding: 7px 12px 8px; border-right: 1px solid var(--line); min-width: 0; } .counter:last-child { border-right: none; } - .counter b { display: block; font-size: 15px; font-weight: 600; letter-spacing: 0.02em; } + .counter b { display: block; font-size: 15px; font-weight: 600; letter-spacing: 0.02em; font-variant-numeric: tabular-nums; } .counter span { font-size: 8px; letter-spacing: 0.12em; color: var(--label); } body.mini header, body.mini #stimulus, body.mini #legend, body.mini #caption, @@ -133,6 +160,19 @@ body.mini #mini-legend { display: flex; } #mini-legend i { display: inline-block; width: 4px; height: 4px; border-radius: 50%; margin-right: 3px; vertical-align: middle; } +
@@ -563,6 +603,7 @@

FLY CONNECTOME · LIVE

const source = new EventSource('/stream'); source.onmessage = (event) => { const frame = JSON.parse(event.data); + document.body.classList.remove('offline'); const now = clock(); if (lastArrival) frameSeconds = frameSeconds * 0.95 + Math.min(now - lastArrival, 0.5) * 0.05; lastArrival = now; @@ -612,7 +653,10 @@

FLY CONNECTOME · LIVE

el('mlamp-r').classList.toggle('on', !!frame.gf_r); el('mlamp-cam').classList.toggle('on', !!frame.camera); }; - source.onerror = () => { el('state').textContent = 'OFFLINE'; }; + source.onerror = () => { + el('state').textContent = 'OFFLINE'; + document.body.classList.add('offline'); + }; } const tl = el('timeline'); @@ -663,7 +707,9 @@

FLY CONNECTOME · LIVE

tctx.fillStyle = color('--label'); tctx.fillText(row.label, 8, y + rowH / 2); tctx.fillStyle = '#101016'; - tctx.fillRect(labelW, y + 1, plotW, rowH - 2); + tctx.beginPath(); + tctx.roundRect(labelW, y + 1, plotW, rowH - 2, 2); + tctx.fill(); const inner = rowH - 4; for (const f of history) { const x = xOf(f.t); diff --git a/src/flynaf/env/brain_view.py b/src/flynaf/env/brain_view.py index 414550a..2bf743a 100644 --- a/src/flynaf/env/brain_view.py +++ b/src/flynaf/env/brain_view.py @@ -5,6 +5,7 @@ import os import struct import time +import urllib.parse import cv2 import numpy as np @@ -189,6 +190,10 @@ def __init__(self, feed: SpikeFeed, highlights: dict, eyes=None): self._page = None self._server = None self._params = config.BRAIN_VIEW + self._page_height = 0 + + def page_height(self) -> int: + return self._page_height async def start(self) -> int: loop = asyncio.get_event_loop() @@ -219,6 +224,9 @@ async def _handle(self, reader: asyncio.StreamReader, writer: asyncio.StreamWrit await self._send(writer, b'application/octet-stream', self._outline) elif path.startswith('/stream'): await self._stream(writer) + elif path.startswith('/viewport'): + self._page_height = _query_int(path, 'h') + await self._send(writer, b'text/plain', b'') else: await self._send(writer, b'text/html; charset=utf-8', _load_page()) except (ConnectionError, asyncio.CancelledError): @@ -273,6 +281,14 @@ async def _stream(self, writer: asyncio.StreamWriter): await asyncio.sleep(interval) +def _query_int(path: str, key: str) -> int: + values = urllib.parse.parse_qs(urllib.parse.urlsplit(path).query).get(key, ['0']) + try: + return max(0, int(values[0])) + except ValueError: + return 0 + + def _load_page() -> bytes: path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'brain_view.html') with open(path, 'rb') as handle: diff --git a/src/flynaf/env/desktop/__init__.py b/src/flynaf/env/desktop/__init__.py index 623f972..ca329da 100644 --- a/src/flynaf/env/desktop/__init__.py +++ b/src/flynaf/env/desktop/__init__.py @@ -88,6 +88,14 @@ def apply_overlay_style(handle: int, x: int, y: int, w: int, h: int) -> bool: return _backend.apply_overlay_style(handle, x, y, w, h) +def outer_rect(handle: int): + return _backend.outer_rect(handle) + + +def clip_window(handle: int, left: int, top: int, right: int, bottom: int, radius: int) -> bool: + return _backend.clip_window(handle, left, top, right, bottom, radius) + + def give_focus_back(handle: int) -> bool: return _backend.give_focus_back(handle) diff --git a/src/flynaf/env/desktop/headless.py b/src/flynaf/env/desktop/headless.py index e40a395..79234de 100644 --- a/src/flynaf/env/desktop/headless.py +++ b/src/flynaf/env/desktop/headless.py @@ -57,6 +57,14 @@ def apply_overlay_style(handle: int, x: int, y: int, w: int, h: int) -> bool: return False +def outer_rect(handle: int): + return None + + +def clip_window(handle: int, left: int, top: int, right: int, bottom: int, radius: int) -> bool: + return False + + def give_focus_back(handle: int) -> bool: return False diff --git a/src/flynaf/env/desktop/linux.py b/src/flynaf/env/desktop/linux.py index 3327c46..43d86eb 100644 --- a/src/flynaf/env/desktop/linux.py +++ b/src/flynaf/env/desktop/linux.py @@ -253,6 +253,14 @@ def apply_overlay_style(handle: int, x: int, y: int, w: int, h: int) -> bool: time.sleep(0.05) +def outer_rect(handle: int): + return None + + +def clip_window(handle: int, left: int, top: int, right: int, bottom: int, radius: int) -> bool: + return False + + def give_focus_back(handle: int) -> bool: window = _window(handle) if window is None: diff --git a/src/flynaf/env/desktop/windows.py b/src/flynaf/env/desktop/windows.py index 67efbe2..f9cc916 100644 --- a/src/flynaf/env/desktop/windows.py +++ b/src/flynaf/env/desktop/windows.py @@ -221,6 +221,24 @@ def apply_overlay_style(handle: int, x: int, y: int, w: int, h: int) -> bool: return placed and on_top and not still_framed +def outer_rect(handle: int): + rect = ctypes.wintypes.RECT() + if not handle or not ctypes.windll.user32.GetWindowRect(handle, ctypes.byref(rect)): + return None + return (rect.left, rect.top, rect.right - rect.left, rect.bottom - rect.top) + + +def clip_window(handle: int, left: int, top: int, right: int, bottom: int, radius: int) -> bool: + gdi32 = ctypes.windll.gdi32 + region = gdi32.CreateRoundRectRgn(left, top, right + 1, bottom + 1, 2 * radius, 2 * radius) + if not region: + return False + if ctypes.windll.user32.SetWindowRgn(handle, region, True): + return True + gdi32.DeleteObject(region) + return False + + def give_focus_back(handle: int) -> bool: if not handle: return False diff --git a/src/flynaf/env/overlay.py b/src/flynaf/env/overlay.py index 0060635..c630c62 100644 --- a/src/flynaf/env/overlay.py +++ b/src/flynaf/env/overlay.py @@ -193,12 +193,51 @@ def pin_as_overlay(window_titles, x: int, y: int, w: int, h: int, return -handle +_PAGE_ROUNDING_PX = 2 + + +def frame_around_page(handle: int, x: int, y: int, w: int, h: int, page_height: int): + client = desktop.window_rect(handle) + outer = desktop.outer_rect(handle) + if client is None or outer is None or page_height <= 0: + return None + left = client[0] - outer[0] + right = outer[0] + outer[2] - (client[0] + client[2]) + bottom = outer[1] + outer[3] - (client[1] + client[3]) + top = outer[3] - bottom - page_height + _PAGE_ROUNDING_PX + if top < 0 or left < 0 or right < 0: + return None + window = (x - left, y - top, w + left + right, h + top + bottom) + page = (left, top, left + w, top + h) + return window, page + + +def strip_browser_chrome(handle: int, x: int, y: int, w: int, h: int, page_height, + radius: int, timeout_sec: float = 20.0): + deadline = time.monotonic() + timeout_sec + frame = previous = None + while frame is None or frame != previous: + if time.monotonic() >= deadline: + return None + previous = frame + time.sleep(0.3) + frame = frame_around_page(handle, x, y, w, h, page_height()) + window, page = frame + if not desktop.apply_overlay_style(handle, *window): + return None + if not desktop.clip_window(handle, *page, radius): + return None + return window, page + + def keep_pinned(handle: int, x: int, y: int, w: int, h: int, stop: object, - interval_sec: float = 2.0): + interval_sec: float = 2.0, clip: tuple | None = None): while not stop.is_set(): if not desktop.is_window(handle): return desktop.apply_overlay_style(handle, x, y, w, h) + if clip is not None: + desktop.clip_window(handle, *clip) stop.wait(interval_sec) diff --git a/src/flynaf/night/panel.py b/src/flynaf/night/panel.py index b5005e7..063da82 100644 --- a/src/flynaf/night/panel.py +++ b/src/flynaf/night/panel.py @@ -22,6 +22,7 @@ pick_overlay_position, pin_as_overlay, screen_size, + strip_browser_chrome, window_title, ) @@ -50,7 +51,7 @@ def restore_game_focus(game: int, panel: int, timeout_sec: float): ) -def _launch_brain_view_window(port: int) -> bool: +def _launch_brain_view_window(port: int, page_height) -> bool: params = config.BRAIN_VIEW url = f'http://127.0.0.1:{port}/' @@ -63,13 +64,13 @@ def _launch_brain_view_window(port: int) -> bool: ) if beside is not None: x, y, width, height = beside - page = url + page = f'{url}?corner={params.corner_radius}' topmost = True _log.info('brain view beside the game, panel %dx%d at %d,%d', width, height, x, y) elif ingame is not None: x, y = ingame width, height = params.ingame_width, params.ingame_height - page = f'{url}?mini=1' + page = f'{url}?mini=1&corner={params.corner_radius}' topmost = True _log.info('brain view in-game overlay, panel %dx%d at %d,%d', width, height, x, y) else: @@ -134,9 +135,18 @@ def _launch_brain_view_window(port: int) -> bool: ) return False + clip = None + stripped = strip_browser_chrome(hwnd, x, y, width, height, page_height, params.corner_radius) + if stripped is None: + _log.warning('the browser title bar could not be cut away, the panel keeps it') + else: + (x, y, width, height), page = stripped + clip = (*page, params.corner_radius) + stop = threading.Event() keeper = threading.Thread( - target=keep_pinned, args=(hwnd, x, y, width, height, stop), daemon=True, + target=keep_pinned, args=(hwnd, x, y, width, height, stop), + kwargs={'clip': clip}, daemon=True, ) keeper.start() _BRAIN_VIEW_PIN['stop'] = stop @@ -158,7 +168,7 @@ async def brain_view_task( _log.info('brain view server listening on %d', port) if config.BRAIN_VIEW.launch_browser: await asyncio.get_event_loop().run_in_executor( - None, _launch_brain_view_window, port + None, _launch_brain_view_window, port, server.page_height ) view_ready.set() await shutdown.wait() diff --git a/tests/test_browser_chrome.py b/tests/test_browser_chrome.py new file mode 100644 index 0000000..069adca --- /dev/null +++ b/tests/test_browser_chrome.py @@ -0,0 +1,35 @@ +from unittest.mock import patch + +import pytest + +from flynaf.env import overlay + +CLIENT = (910, 200, 260, 140) +OUTER = (900, 200, 280, 150) + + +def _frame(page_height): + with patch.object(overlay.desktop, 'window_rect', return_value=CLIENT), \ + patch.object(overlay.desktop, 'outer_rect', return_value=OUTER): + return overlay.frame_around_page(1, 900, 200, 280, 150, page_height) + + +def test_the_title_bar_ends_up_above_the_panel_and_outside_the_clip(): + window, page = _frame(98) + + title_bar = 140 - 98 + top = title_bar + overlay._PAGE_ROUNDING_PX + assert window == (890, 200 - top, 300, 150 + top + 10) + assert page == (10, top, 290, top + 150) + + +def test_the_clip_lands_on_the_requested_rectangle(): + window, page = _frame(98) + + assert (window[0] + page[0], window[1] + page[1]) == (900, 200) + assert (page[2] - page[0], page[3] - page[1]) == (280, 150) + + +@pytest.mark.parametrize('page_height', [0, 200]) +def test_a_page_height_that_does_not_fit_the_window_is_refused(page_height): + assert _frame(page_height) is None From 53de7a3821ac116eef04216b1e39a0ae14554ec4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Artur=20Mendon=C3=A7a=20Arruda?= Date: Thu, 1 Oct 2026 21:36:23 -0300 Subject: [PATCH 10/14] feat(sim): start the night from a settled brain on one clock --- ...tarts-the-night-once-the-brain-is-ready.md | 4 ++ ...a-night-reads-one-high-resolution-clock.md | 66 +++++++++++++++++++ run.py | 12 +++- src/flynaf/clock.py | 3 + src/flynaf/config.py | 3 + src/flynaf/env/desktop/windows.py | 8 ++- src/flynaf/env/tablet_feed.py | 6 +- src/flynaf/night/engine.py | 21 ++++++ src/flynaf/night/monitor.py | 7 +- src/flynaf/night/tasks.py | 30 ++++++--- src/flynaf/replay.py | 1 + src/flynaf/scripts/debug_tablet.py | 8 +-- src/flynaf/telemetry.py | 40 +++++++---- tests/test_network_warmup.py | 40 +++++++++++ tests/test_search_loop_wiring.py | 5 ++ 15 files changed, 217 insertions(+), 37 deletions(-) create mode 100644 docs/adr/0028-a-night-reads-one-high-resolution-clock.md create mode 100644 src/flynaf/clock.py create mode 100644 tests/test_network_warmup.py diff --git a/docs/adr/0021-the-launcher-starts-the-night-once-the-brain-is-ready.md b/docs/adr/0021-the-launcher-starts-the-night-once-the-brain-is-ready.md index 884d201..eb90ab1 100644 --- a/docs/adr/0021-the-launcher-starts-the-night-once-the-brain-is-ready.md +++ b/docs/adr/0021-the-launcher-starts-the-night-once-the-brain-is-ready.md @@ -112,6 +112,9 @@ browser tab open on this repository's page carries the name in its title, and it made the lookup report a running game, so the launcher skipped opening it and went straight to the brain. The title fallback now requires the whole title to be the game's, and the process name stays the primary test. +Being ready also has to mean that the brain has stopped starting up. Almost every recorded night opened with a door slam the eye did not cause. Of course it was weird, in seven of the eight traces long enough to tell, DNp01 fired within the first ten engine frames with both eye populations at zero input. The engine used to take its first step only once calibration had finished, from `reset_state`, which puts all 138,639 neurons at `v0` with no conductance. Stepped from there with only the background noise, the network recruits about 150 active neurons per frame up to about 3,000, excitation outruns the inhibition that later balances it, and over four measured starts the Giant Fiber fired inside that wave three times, always between frames 10 and 17, and never after frame 20. A real fly has no such moment, because its brain was running long before it was put in front of the hallways. The engine now steps without input while calibration runs, until the neurons active in the last ten frames are no more than 1.2 times those in the ten before, with a cap of 200 frames, and nothing in that warm-up reaches a motor path. The measured wave first meets the criterion around frame 30. Over six fresh starts the network settled after 29 to 35 frames and the Giant Fiber stayed silent through the 150 unstimulated frames that followed each one. The session report prints the warm-up, and `replay` settles the engine the same way before it feeds a recorded night. + +The window has to be measured in physical pixels too. Windows reports window rectangles scaled down by the display setting to any process that has not declared itself DPI aware, and the declaration used to happen as a side effect of the first `screen_size()` call. On a display at 150 percent, a run started with `--manual` measured the game at 213,131 sized 853x480 instead of 320,197 sized 1280x720 and placed the panel "beside" a game that actually extended under it, over the right eye, the right door and light buttons and part of the camera map. The Windows backend now declares DPI awareness when it is loaded, before anything reads a coordinate. | Version | Description | Author(s) | Date | Reviewer(s) | Review Date | |---|---|---|---|---|---| @@ -121,3 +124,4 @@ the game's, and the process name stays the primary test. | 1.3 | Recorded the profile sweep, the sync bubble and the Linux parent death signal | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-28 | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-28 | | 1.4 | Replaced the drawn head with the NeuroMechFly model in a night office scene | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-29 | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-29 | | 1.5 | Required the whole window title to match the game | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-30 | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-30 | +| 1.6 | Warmed the network up before the night and declared DPI awareness before the first window measurement | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-10-01 | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-10-01 | diff --git a/docs/adr/0028-a-night-reads-one-high-resolution-clock.md b/docs/adr/0028-a-night-reads-one-high-resolution-clock.md new file mode 100644 index 0000000..add5503 --- /dev/null +++ b/docs/adr/0028-a-night-reads-one-high-resolution-clock.md @@ -0,0 +1,66 @@ +# ADR 0028: A Night Reads One High Resolution Clock + +## Status +Accepted. + +## Context +The time of a night came from `time.time()`. The engine loop read it once per +frame, the vision loop read it before every capture, and every door, saccade and +tablet deadline was written against it. The frame budget alone was measured with +`time.perf_counter()`. + +`time.time()` is the wall clock. It can be adjusted while a night runs, and on +Windows its resolution is that of the system timer, not of the processor. On the +project's Python 3.11 the three clocks resolve very different intervals. + +| Clock | Implementation on Windows | Resolution | +|---|---|---| +| `time.time` | GetSystemTimeAsFileTime | 15.6 ms nominal | +| `time.monotonic` | GetTickCount64 | 15.6 ms | +| `time.perf_counter` | QueryPerformanceCounter | 0.1 us | + +The wall clock ticked every millisecond while the measurement ran, because +another process had raised the timer resolution, and it ticks every 15.6 ms when +nothing has. The fly's behaviour therefore depended on what else was open. + +Every leak in the loop is scaled by the measured time between frames. The door +hold decays as `hold_leak ** (frame_elapsed * fps)`, and DNp09 and the search +drive integrate over the same interval. At ten frames a second a quantum of +15.6 ms is a sixth of a frame. The tablet is worse. `LoomChannel` derives the +speed it feeds LC4 from the change in contrast over the time between two reads +taken 30 ms apart, so an interval read as one or two quanta, or as zero and +clamped to a millisecond, scales the drive by the same factor. + +`time.monotonic` is the usual answer to an adjustable clock, but on this Python +it has the same 15.6 ms resolution and would not help. Python 3.13 moves it to +QueryPerformanceCounter. + +## Decision +`flynaf/clock.py` exposes `now`, bound to `time.perf_counter`, and every reading +of time that a night acts on goes through it. That covers the vision and engine +loops, the hallway observation and the saccade refractory period, the office +recovery in the monitor, the tablet reference capture, the session telemetry +and the live tablet debug script. The engine loop measures its frame budget from +the same reading it hands to the engine, so the loop no longer reads two clocks. + +Wall clock time stays where a person reads it, the timestamps in file names and +in the session report. + +## Consequences +The intervals the leaks and the loom speed are computed from are exact to well +under a microsecond whatever the system timer is doing, and a clock adjustment +during a night no longer moves a deadline. + +All of a night's times have to come from the same clock, because a door sets +`blind_until` from the engine's reading and the vision loop compares it with its +own. A new loop that reads `time.time()` directly would compare times taken on +two different clocks without any error to show for it. + +The deadlines outside a night, the launcher, the overlay placement and the +Linux window settling, stay on `time.monotonic`. They only compare a reading +with another reading of the same clock, at a resolution far finer than the +seconds they wait. + +| Version | Description | Author(s) | Date | Reviewer(s) | Review Date | +|---|---|---|---|---|---| +| 1.0 | Recorded the move of every time step in a night to one perf_counter clock | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-10-01 | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-10-01 | diff --git a/run.py b/run.py index 40e9399..bb6f4cc 100644 --- a/run.py +++ b/run.py @@ -2,6 +2,7 @@ import glob import logging import os +from datetime import datetime from flynaf import config, launcher from flynaf.env import desktop @@ -95,8 +96,17 @@ def cmd_play(mode: str | None) -> int: return 0 +def _log_handlers() -> list: + folder = os.path.join(config.PROJECT_ROOT, 'logs') + os.makedirs(folder, exist_ok=True) + stamp = datetime.now().strftime('%Y%m%d_%H%M%S') + path = os.path.join(folder, f'run_{stamp}.log') + return [logging.StreamHandler(), logging.FileHandler(path, encoding='utf-8')] + + def main() -> int: - logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s') + logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s', + handlers=_log_handlers()) parser = argparse.ArgumentParser(description='Run Fly-NAF, or one of the ways to inspect it.') parser.add_argument('--verify', action='store_true', help='check the connectome and exit') parser.add_argument('--replay', nargs='?', const='', metavar='TRACE', diff --git a/src/flynaf/clock.py b/src/flynaf/clock.py new file mode 100644 index 0000000..a153cbf --- /dev/null +++ b/src/flynaf/clock.py @@ -0,0 +1,3 @@ +import time + +now = time.perf_counter diff --git a/src/flynaf/config.py b/src/flynaf/config.py index 09d9289..4c4ec42 100644 --- a/src/flynaf/config.py +++ b/src/flynaf/config.py @@ -48,6 +48,9 @@ class SimulationParams: steps_per_frame: int = 2 target_fps: int = 10 arousal_multiplier: float = 3.0 + settle_window_frames: int = 10 + settle_growth_ratio: float = 1.2 + settle_max_frames: int = 200 @dataclass(frozen=True) diff --git a/src/flynaf/env/desktop/windows.py b/src/flynaf/env/desktop/windows.py index f9cc916..16dde34 100644 --- a/src/flynaf/env/desktop/windows.py +++ b/src/flynaf/env/desktop/windows.py @@ -38,6 +38,13 @@ _KEYEVENTF_KEYUP = 0x0002 +def _read_physical_pixels(): + ctypes.windll.user32.SetProcessDPIAware() + + +_read_physical_pixels() + + def _user32(): user32 = ctypes.windll.user32 user32.SetWindowPos.argtypes = [ @@ -116,7 +123,6 @@ def find_browser(): def screen_size() -> tuple: user32 = ctypes.windll.user32 - user32.SetProcessDPIAware() return (user32.GetSystemMetrics(0), user32.GetSystemMetrics(1)) diff --git a/src/flynaf/env/tablet_feed.py b/src/flynaf/env/tablet_feed.py index 326ccf8..ddd8821 100644 --- a/src/flynaf/env/tablet_feed.py +++ b/src/flynaf/env/tablet_feed.py @@ -7,7 +7,7 @@ import mss import numpy as np -from flynaf import config +from flynaf import clock, config from flynaf.env import anchor _log = logging.getLogger(__name__) @@ -83,8 +83,8 @@ def capture_reference(self, camera: str) -> bool: return True def _collect(self, seconds: float) -> list: - frames, deadline, last = [], time.time() + seconds, None - while time.time() < deadline: + frames, deadline, last = [], clock.now() + seconds, None + while clock.now() < deadline: with self._lock: frame = self._latest[-1] if self._latest else None if frame is not None and frame is not last: diff --git a/src/flynaf/night/engine.py b/src/flynaf/night/engine.py index 62a7b10..fd41b09 100644 --- a/src/flynaf/night/engine.py +++ b/src/flynaf/night/engine.py @@ -7,6 +7,14 @@ from flynaf.night.state import SensoryState +def has_settled(activity: list, window: int, growth_ratio: float) -> bool: + if len(activity) < 2 * window: + return False + earlier = sum(activity[-2 * window:-window]) + recent = sum(activity[-window:]) + return recent <= earlier * growth_ratio + + class ConnectomeEngine: def __init__(self, device: str, cell_index: CellTypeIndex | None = None): self._device = device @@ -29,6 +37,7 @@ def __init__(self, device: str, cell_index: CellTypeIndex | None = None): self.explore_membrane = 0.0 self.frames = 0 self.driven_frames = {'left': 0, 'right': 0} + self.warmup_activity = [] @property def sensory_span(self) -> int: @@ -54,6 +63,18 @@ def step(self, state: SensoryState, current_time: float) -> torch.Tensor: def reset(self): self._adapter.model.reset_state() + def settle(self) -> bool: + params = config.SIMULATION_PARAMS + quiet = SensoryState() + self.warmup_activity = [] + while len(self.warmup_activity) < params.settle_max_frames: + self._load_rates(quiet) + spikes = self._adapter.step(self._rates, steps=self._steps) + self.warmup_activity.append(int(spikes.count_nonzero())) + if has_settled(self.warmup_activity, params.settle_window_frames, params.settle_growth_ratio): + return True + return False + def _load_rates(self, state: SensoryState): self._rates.zero_() for indices, rate_hz in stimulus_rates(self.neurons, state, self._base_rate): diff --git a/src/flynaf/night/monitor.py b/src/flynaf/night/monitor.py index 9c3485f..e5314c1 100644 --- a/src/flynaf/night/monitor.py +++ b/src/flynaf/night/monitor.py @@ -1,8 +1,7 @@ import asyncio import logging -import time -from flynaf import config +from flynaf import clock, config from flynaf.env.input_controller import FNAFController from flynaf.env.vision import FNAFVision from flynaf.night.motor import await_motor @@ -16,9 +15,9 @@ async def office_comes_back( vision: FNAFVision, controller, settle_sec: float, confirm_sec: float ) -> bool: vision.clear_buffers() - deadline = time.time() + confirm_sec + deadline = clock.now() + confirm_sec agreed = 0 - while time.time() < deadline: + while clock.now() < deadline: await asyncio.sleep(settle_sec / 2.0) if controller is not None and controller.facing() != 'centre': agreed = 0 diff --git a/src/flynaf/night/tasks.py b/src/flynaf/night/tasks.py index 38ec833..e772fc6 100644 --- a/src/flynaf/night/tasks.py +++ b/src/flynaf/night/tasks.py @@ -1,8 +1,7 @@ import asyncio import logging -import time -from flynaf import config +from flynaf import clock, config from flynaf.env.brain_view import SpikeFeed from flynaf.env.input_controller import FNAFController from flynaf.env.vision import FNAFVision @@ -30,7 +29,7 @@ async def vision_task( while not shutdown.is_set(): state.office_centred = controller.facing() == 'centre' cam_up = state.office_centred and vision.is_camera_up() - now = time.time() + now = clock.now() looking_left = state.check_left and now >= state.blind_until['left'] looking_right = state.check_right and now >= state.blind_until['right'] state.left_rate = vision.get_left_sensory_rate() if looking_left else 0.0 @@ -47,7 +46,7 @@ async def observe_hallway( vision.reset_peak_mse(side) opened_at_frame = engine.frames opened_at_driven = engine.driven_frames[side] - deadline = time.time() + foraging.light_inspection_max_sec + deadline = clock.now() + foraging.light_inspection_max_sec if side == 'left': state.check_left = True @@ -59,7 +58,7 @@ async def observe_hallway( if state.l_spike if side == 'left' else state.r_spike: fired = True break - if time.time() >= deadline: + if clock.now() >= deadline: break await asyncio.sleep(engine.frame_dt / 2) @@ -116,7 +115,7 @@ async def saccade_task( ', giant fiber answered' if fired else '') telemetry.record_look_contrast(side, contrast, frames, driven) - saccade.ready_at = time.time() + config.FORAGING_PARAMS.saccade_refractory_sec + saccade.ready_at = clock.now() + config.FORAGING_PARAMS.saccade_refractory_sec saccade.side = None saccade.busy = False @@ -135,6 +134,7 @@ async def engine_task( trace: SessionRecorder | None = None, tablet=None, ): + await warm_up(engine, telemetry) await calibration_done.wait() trace = trace or SessionRecorder(enabled=False) tablet = tablet or IdleTablet() @@ -144,11 +144,10 @@ async def engine_task( monitor = MonitorControl(vision, controller, telemetry, state, tablet) doors = DoorControl(controller, telemetry, state) watchdog = InhibitionWatchdog(vision) - last_frame_at = time.time() + last_frame_at = clock.now() while not shutdown.is_set(): - t_start = time.perf_counter() - now = time.time() + now = clock.now() frame_elapsed = now - last_frame_at last_frame_at = now @@ -175,10 +174,21 @@ async def engine_task( doors.release_decayed(now, frame_elapsed) answer_giant_fibers(engine, doors, feed, state, now) - remaining = engine.frame_dt - (time.perf_counter() - t_start) + remaining = engine.frame_dt - (clock.now() - now) await asyncio.sleep(max(remaining, 0.0)) +async def warm_up(engine: ConnectomeEngine, telemetry: ConnectomeTelemetry): + settled = await asyncio.to_thread(engine.settle) + frames = len(engine.warmup_activity) + if settled: + _log.info('the network settled after %d warm-up frames, %d neurons active per frame', + frames, engine.warmup_activity[-1]) + else: + _log.warning('the network was still growing after %d warm-up frames, starting anyway', frames) + telemetry.record_warmup(frames, settled, engine.warmup_activity) + + def read_spikes(engine, spikes, state: SensoryState): frame = spikes[0] state.l_spike = bool(frame[engine.l_motor_idx].any()) diff --git a/src/flynaf/replay.py b/src/flynaf/replay.py index e781755..43fb21d 100644 --- a/src/flynaf/replay.py +++ b/src/flynaf/replay.py @@ -43,6 +43,7 @@ def replay(trace_path: str, device: str = 'cpu') -> dict: _log.info('replaying %d frames from %s', len(frames), os.path.basename(trace_path)) engine = ConnectomeEngine(device) + engine.settle() state = SensoryState() left_driven, right_driven = [], [] diff --git a/src/flynaf/scripts/debug_tablet.py b/src/flynaf/scripts/debug_tablet.py index 2964e50..9d84260 100644 --- a/src/flynaf/scripts/debug_tablet.py +++ b/src/flynaf/scripts/debug_tablet.py @@ -1,7 +1,7 @@ import argparse import time -from flynaf import config +from flynaf import clock, config from flynaf.env.overlay import anchor_to_game from flynaf.env.tablet_feed import TabletFeed from flynaf.night.state import SensoryState @@ -27,10 +27,10 @@ def main() -> int: state = SensoryState(camera_open=True, tablet_camera=args.camera) senses = TabletSenses(feed) print('watching, bring something into view. Ctrl+C stops.') - deadline = time.time() + args.seconds + deadline = clock.now() + args.seconds try: - while time.time() < deadline: - now = time.time() + while clock.now() < deadline: + now = clock.now() contrast = feed.contrast(args.camera) or 0.0 levels = senses.read(state, now) shown = ' '.join(f'{name} {level:.2f}' for name, level in levels.items()) diff --git a/src/flynaf/telemetry.py b/src/flynaf/telemetry.py index a23df9a..9f6c5c2 100644 --- a/src/flynaf/telemetry.py +++ b/src/flynaf/telemetry.py @@ -1,14 +1,13 @@ import os -import time from datetime import datetime -from flynaf import config +from flynaf import clock, config REFLEX_FRAMES = 10 class ConnectomeTelemetry: def __init__(self): - self.start_time = time.time() + self.start_time = clock.now() self.events = [] self.stats = { 'left_door_panics': 0, @@ -36,14 +35,16 @@ def __init__(self): self.monitor_stuck = 0 self.saccade_gaps = [] self._last_saccade_at = None + self._first_frame_at = None + self.warmup = None def record_door_panic(self, side: str, cause: str = 'giant fiber'): - t = time.time() - self.start_time + t = clock.now() - self.start_time self.stats[f'{side}_door_panics'] += 1 self.events.append(f"[{t:>6.1f}s] Visual threat detected. {cause.capitalize()} fired. {side.capitalize()} door slammed.") def record_camera_view(self, camera: str, cause: str): - t = time.time() - self.start_time + t = clock.now() - self.start_time self.camera_views[camera] = self.camera_views.get(camera, 0) + 1 if cause in self.camera_view_causes: self.camera_view_causes[cause] += 1 @@ -51,7 +52,7 @@ def record_camera_view(self, camera: str, cause: str): self.events.append(f"[{t:>6.1f}s] Watching CAM {camera}, {why}.") def record_tablet_escape(self, side: str, camera: str): - t = time.time() - self.start_time + t = clock.now() - self.start_time key = f'{camera} {side}' self.tablet_escapes[key] = self.tablet_escapes.get(key, 0) + 1 self.events.append( @@ -59,13 +60,13 @@ def record_tablet_escape(self, side: str, camera: str): ) def record_door_release(self, side: str, held_sec: float): - t = time.time() - self.start_time + t = clock.now() - self.start_time self.stats[f'{side}_door_releases'] += 1 self.door_holds.append(held_sec) self.events.append(f"[{t:>6.1f}s] Escape drive decayed after {held_sec:.1f}s. {side.capitalize()} door reopened.") def record_camera_pull(self, commanded: bool, drive: float): - t = time.time() - self.start_time + t = clock.now() - self.start_time self.stats['camera_pulls'] += 1 self.camera_causes['spike' if commanded else 'drive'] += 1 how = ('DNp09 fired outright' if commanded @@ -80,7 +81,7 @@ def record_camera_pull(self, commanded: bool, drive: float): } def record_camera_release(self, watched_sec: float, reason: str): - t = time.time() - self.start_time + t = clock.now() - self.start_time self.camera_watches.append(watched_sec) if reason in self.camera_release_reasons: self.camera_release_reasons[reason] += 1 @@ -89,7 +90,12 @@ def record_camera_release(self, watched_sec: float, reason: str): f"[{t:>6.1f}s] Monitor lowered after {watched_sec:.1f}s, {why}." ) + def record_warmup(self, frames: int, settled: bool, activity: list): + self.warmup = (frames, settled, activity[-1] if activity else 0) + def record_frame(self, inhibited: bool, looking: bool): + if self._first_frame_at is None: + self._first_frame_at = clock.now() self.engine_frames += 1 if inhibited: self.inhibited_frames += 1 @@ -102,7 +108,7 @@ def record_look_contrast(self, side: str, mse: float, frames: int, driven: int): self.look_driven.append(driven) def record_monitor_stuck(self): - t = time.time() - self.start_time + t = clock.now() - self.start_time self.monitor_stuck += 1 self.events.append( f"[{t:>6.1f}s] Monitor stayed up after both lowering gestures. Retrying." @@ -115,11 +121,11 @@ def record_monitor_stuck(self): } def record_light_saccade(self, side: str, drive: float, reason: str): - t = time.time() - self.start_time + t = clock.now() - self.start_time self.stats[f'{side}_light_saccades'] += 1 if reason in self.saccade_reasons: self.saccade_reasons[reason] += 1 - now = time.time() + now = clock.now() if self._last_saccade_at is not None: self.saccade_gaps.append(now - self._last_saccade_at) self._last_saccade_at = now @@ -141,7 +147,7 @@ def _write_tablet(self, f): f.write(f"{'DNp04 escapes from the tablet':<38}: {escapes}\n\n") def dump_report(self): - duration = time.time() - self.start_time + duration = clock.now() - self.start_time os.makedirs("logs", exist_ok=True) timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") filepath = os.path.join("logs", f"session_telemetry_{timestamp}.txt") @@ -190,8 +196,14 @@ def dump_report(self): self._write_tablet(f) f.write("--- Where The Night Went ---\n") + if self.warmup is not None: + frames, settled, active = self.warmup + verdict = 'settled' if settled else 'still growing, the cap ended it' + f.write(f"{'Network warm-up before the night':<38}: " + f"{frames} frames, {verdict}, {active} neurons active per frame\n") if self.engine_frames: - rate = self.engine_frames / max(duration, 1.0) + running = clock.now() - (self._first_frame_at or self.start_time) + rate = self.engine_frames / max(running, 1.0) f.write(f"{'Engine rate':<38}: {rate:.1f} frames/sec, " f"{self.engine_frames} frames\n") f.write(f"{'Inhibitors driven':<38}: " diff --git a/tests/test_network_warmup.py b/tests/test_network_warmup.py new file mode 100644 index 0000000..5d3df8a --- /dev/null +++ b/tests/test_network_warmup.py @@ -0,0 +1,40 @@ +import pytest + +from flynaf import config +from flynaf.night.engine import has_settled + +WINDOW = config.SIMULATION_PARAMS.settle_window_frames +RATIO = config.SIMULATION_PARAMS.settle_growth_ratio + +MEASURED_RECRUITMENT = [ + 153, 153, 131, 156, 156, 186, 259, 278, 387, 527, 569, 778, 932, 1428, 1406, 2434, + 2270, 2685, 2224, 2607, 2383, 2692, 2339, 2907, 2200, 2700, 2391, 2872, 2446, 2925, + 2399, 3191, 2702, 3070, 2904, 3430, 2775, 3299, 2855, 3317, +] +LAST_SPURIOUS_GIANT_FIBER_FRAME = 17 + + +def first_settled_frame(activity: list) -> int | None: + for end in range(1, len(activity) + 1): + if has_settled(activity[:end], WINDOW, RATIO): + return end + return None + + +@pytest.mark.parametrize('activity, expected', [ + ([], False), + ([3000] * (2 * WINDOW - 1), False), + ([3000] * (2 * WINDOW), True), + ([150] * WINDOW + [3000] * WINDOW, False), + ([3000] * WINDOW + [2000] * WINDOW, True), +]) +def test_settling_needs_two_full_windows_without_growth(activity, expected): + assert has_settled(activity, WINDOW, RATIO) is expected + + +def test_the_measured_recruitment_wave_is_over_before_the_network_counts_as_settled(): + settled_at = first_settled_frame(MEASURED_RECRUITMENT) + + assert settled_at is not None + assert settled_at > LAST_SPURIOUS_GIANT_FIBER_FRAME + assert settled_at < config.SIMULATION_PARAMS.settle_max_frames diff --git a/tests/test_search_loop_wiring.py b/tests/test_search_loop_wiring.py index 8cd24cf..976fc5f 100644 --- a/tests/test_search_loop_wiring.py +++ b/tests/test_search_loop_wiring.py @@ -45,8 +45,13 @@ def __init__(self): self.r_escape_idx = [6] self.frames = 0 self.driven_frames = {'left': 0, 'right': 0} + self.warmup_activity = [] self._t0 = None + def settle(self): + self.warmup_activity = [NEURONS] + return True + def step(self, state, now): self.frames += 1 if self._t0 is None: From ebb9fe0c74a13ee078496ff308e838a923d6e496 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Artur=20Mendon=C3=A7a=20Arruda?= Date: Thu, 1 Oct 2026 21:54:04 -0300 Subject: [PATCH 11/14] feat(night): read the hallways, the tablet and the closed doors from the screen --- .../0001-opencv-brightness-thresholding.md | 32 +- ...ors-reopen-when-the-escape-drive-decays.md | 42 ++- ...comes-down-only-when-the-office-is-back.md | 37 +-- ...ght-is-a-package-and-main-only-wires-it.md | 3 +- ...the-lobula-and-dnp04-answers-through-it.md | 5 +- ...14-the-shadow-that-looked-like-the-dark.md | 41 +++ src/flynaf/config.py | 36 ++- src/flynaf/env/brain_view.py | 39 ++- src/flynaf/env/hallway_bank.py | 22 ++ src/flynaf/env/input_controller.py | 58 ++-- src/flynaf/env/overlay.py | 7 +- src/flynaf/env/tablet_map.py | 48 +++ src/flynaf/env/vision.py | 297 ++++++++++-------- src/flynaf/main.py | 5 +- src/flynaf/night/calibration.py | 55 +++- src/flynaf/night/doors.py | 65 +++- src/flynaf/night/lights.py | 28 ++ src/flynaf/night/monitor.py | 189 ++++++----- src/flynaf/night/motor.py | 2 +- src/flynaf/night/panel.py | 8 +- src/flynaf/night/state.py | 5 +- src/flynaf/night/tablet/references.py | 33 +- src/flynaf/night/tasks.py | 123 ++++---- src/flynaf/scripts/debug_camera.py | 54 +--- src/flynaf/scripts/debug_reflex.py | 8 +- src/flynaf/scripts/measure_door_window.py | 72 +++++ src/flynaf/telemetry.py | 44 ++- src/flynaf/tuning.py | 7 +- tests/test_camera_reference_refresh.py | 74 ----- tests/test_closed_door_window.py | 92 ++++++ tests/test_corollary_discharge.py | 57 +--- tests/test_door_release.py | 8 +- tests/test_door_verification.py | 228 ++++++++++++++ tests/test_hallway_detection.py | 92 +++--- tests/test_light_check.py | 89 ++++++ tests/test_monitor_lowering.py | 264 ++++++++-------- tests/test_motor_pan_settle.py | 5 +- tests/test_overlay_placement.py | 9 + tests/test_search_loop_wiring.py | 54 ++-- tests/test_startup_readiness.py | 25 +- tests/test_tablet_escape.py | 64 ++-- tests/test_tablet_map_detection.py | 70 +++++ tests/test_vision_buffers.py | 59 +--- tests/test_vision_reference_cache.py | 60 ++-- 44 files changed, 1651 insertions(+), 964 deletions(-) create mode 100644 docs/experiments/14-the-shadow-that-looked-like-the-dark.md create mode 100644 src/flynaf/env/hallway_bank.py create mode 100644 src/flynaf/env/tablet_map.py create mode 100644 src/flynaf/night/lights.py create mode 100644 src/flynaf/scripts/measure_door_window.py delete mode 100644 tests/test_camera_reference_refresh.py create mode 100644 tests/test_closed_door_window.py create mode 100644 tests/test_door_verification.py create mode 100644 tests/test_light_check.py create mode 100644 tests/test_tablet_map_detection.py diff --git a/docs/adr/0001-opencv-brightness-thresholding.md b/docs/adr/0001-opencv-brightness-thresholding.md index b14dacd..864d74a 100644 --- a/docs/adr/0001-opencv-brightness-thresholding.md +++ b/docs/adr/0001-opencv-brightness-thresholding.md @@ -1,26 +1,32 @@ # 0001. Use Grayscale Brightness Thresholding for Threat Detection ## Status -Accepted +Accepted. Amended on 2026-10-01, when the single empty-hallway reference and its MSE threshold were replaced by a bank of calibrated views. ## Context -The project requires detecting the appearance of an animatronic (Bonnie) at the left door in the FNAF 1 game environment. Initial attempts relied on capturing the exact RGB color signature of the animatronic. However, dynamic video rendering and screen capturing can distort RGB values (e.g., color compression, shifts in purple hues, and BGR/RGB inversions in the `mss` library). Additionally, performing full-screen Object Detection (like YOLO) would consume heavy CPU cycles, competing directly with the 138,000-neuron PyTorch biological simulation. -I needed a detection mechanism that is CPU-agnostic, immune to minor color shifts, and highly precise. +The project requires detecting an animatronic at either door in FNAF 1. Exact RGB signatures did not survive screen capture, with compression, purple shifts and the BGR order of `mss` all moving the values, and a full-screen object detector would have competed for CPU with the 138,000-neuron simulation. The first version read the mean brightness of a 10x10 box at Bonnie's spawn point. It was later replaced by the mean squared error between each hallway patch and a reference of that hallway captured empty with the light on, with a threat declared above 1500. + +That threshold could never see an animatronic. A live night on 2026-10-01 recorded the exact frames the eye compared while Bonnie stood lit in the left doorway, and he was 124 from the empty reference. The game is dark, Bonnie is a dark figure on a dark background, and the MSE averages his difference over the whole 481x481 patch. Every detection the project had logged above 1500 turned out to be something bright and large covering the eye, the camera tablet, the game over static or another window. The same recording showed that the hallway light flickers, and a flicker frame sits 149 from the lit reference, further than Bonnie. No threshold on the distance to one reference separates the two. + +The recording also showed that FNAF draws each hallway from a small fixed set of pre-rendered images. An empty lit hallway matched its reference at exactly zero on every frame, the flicker frame was always the same image, and on the right side the flicker frame is the light-off frame. Bonnie was 118 from the nearest of those views. + +The eye also depended on a light whose state nobody checked. The controller toggled each light from what it believed, so a light that was already on when the night began was switched off by the click meant to switch it on, and the fly then looked at a dark hallway for the rest of the night without knowing. ## Decision -I will use OpenCV grayscale brightness thresholding (measuring `np.mean(gray)`) limited to a hardcoded 10x10 pixel bounding box targeted exactly at the animatronic's spawn coordinate. If the brightness exceeds 50.0 points, the system registers a threat. -Because the FNAF hallway spotlight does not directly illuminate this specific edge pixel when the hall is empty, the brightness remains near 0 even when the light is turned on. It only spikes when the animatronic physically steps into the light cone and reflects light onto the target pixel. +Each hallway is compared against a bank of views instead of one reference. The live calibration, run at midnight when both hallways are empty, records each side with the door open and with the door closed, first with the light on for `reference_sweep_sec`, long enough to catch the flicker, and then with the light off. A frame enters the bank only when it differs from every frame already there by more than `bank_tolerance_mse`. The banks are cached together with an index and the layout fingerprint, and a cache without them forces a new calibration. -## Consequences -**Positive:** -- CPU usage for vision is effectively 0%, leaving maximum resources for PyTorch. -- Immune to RGB color shifts, BGR library bugs, and video rendering artifacts. -- Immune to false positives from just turning on the light (due to the shadow zone calibration). +The distance of a frame is its MSE to the nearest view in the bank for the door's current state, and the evidence for a hallway is the median of those distances over the frames in the buffer, so one odd frame cannot raise or hide a threat. The threshold is 40, a third of the 118 measured for Bonnie and far above the zero every empty view reads, and the tuning range stops at 100 so it cannot be set where Bonnie would disappear again. Each frame's distance is computed once and reused while it stays in the buffer. + +The light button is read off the screen. It measured 93 to 97 in grey level off and 186 to 196 on, and the fly treats it as lit above 140. After asking for a light, the fly reads the button twice 80 ms apart and presses it again only when both readings disagree, at most twice, and the press records the state the screen confirmed rather than flipping the controller's belief. A light that will not come on is logged, which in FNAF 1 means an animatronic is already in the office or the power is gone. -**Negative:** -- Highly fragile to in-game camera panning or window resizing. If the absolute coordinates shift by even a few pixels, the system will target a dead zone and fail. -- Requires recalibration if the user switches monitors or display scaling. +## Consequences +- The eye detected Bonnie live at 118 against the new threshold and closed the left door, the first animatronic among all the threat images the project has saved. Replayed through the new code, the recorded frames gave 17 detections out of 17 windows with Bonnie in the doorway and none out of 52 empty windows. +- The first night after the change spends about 70 seconds calibrating with both doors briefly closed and no hallway watched, which is long enough on a continued night for an animatronic to walk in. One recorded night was lost that way. +- The banks are images of the operator's own game, kept under `logs/` and never committed. +- Chica in the right window and Bonnie's shadow behind a closed left door have not been measured against the bank yet. Both thresholds start at 40, and `measure_door_window.py` measures the closed views. +- The light check adds up to a few hundred milliseconds to a look when a correction is needed and nothing when the light is where the controller believes it is. | Version | Description | Author(s) | Date | Reviewer(s) | Review Date | |---------|-------------|-----------|------|-------------|-------------| | 1.0 | Initial version | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-02 | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-02 | +| 1.1 | Replaced the single reference with a bank of calibrated views, measured the threshold against Bonnie, and checked the light on screen | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-10-01 | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-10-01 | diff --git a/docs/adr/0011-doors-reopen-when-the-escape-drive-decays.md b/docs/adr/0011-doors-reopen-when-the-escape-drive-decays.md index 33f9fe8..3b55826 100644 --- a/docs/adr/0011-doors-reopen-when-the-escape-drive-decays.md +++ b/docs/adr/0011-doors-reopen-when-the-escape-drive-decays.md @@ -1,33 +1,49 @@ # ADR 0011: Doors Reopen When the Escape Drive Decays ## Status -Accepted. +Accepted. Amended on 2026-10-01, when the eye stopped being blind behind a closed door and reopening started to wait for a look. ## Context Until now a door only ever closed. `close_left_door` set `door_state['left'] = True` and nothing ever set it back, so the first Giant Fiber spike of the night shut that door permanently. In FNAF a closed door draws power continuously, so a fly that slams both doors early has already lost the night at around 3 AM regardless of what the connectome does afterwards. The reflex was complete and the recovery was missing. -The naive fix, reopening once the hallway looks clear again, does not work here, and the reason is in `_vision_task`: the eye only reads the doorway while a light saccade is running, and it reads it as an MSE deviation from a reference captured on an empty, lit hallway. With the door shut, that same region is a door panel, which deviates from the reference far more than any animatronic does. The fly would read its own closed door as a permanent threat, keep the Giant Fiber charged, and never release. Re-referencing against a "door closed" frame inverts the problem rather than solving it, because the reference would then be captured with the animatronic still standing there, and its departure would register as the change. +The first version of this decision reopened the door on a timer-like trace and kept the eye blind behind it. The argument was that the eye reads the doorway as an MSE deviation from a reference captured on an empty, lit hallway, that a shut door deviates from that reference far more than any animatronic does, and that a closed door blocks the view anyway. Re-referencing against a "door closed" frame looked like it would only invert the problem, because that frame would be captured with the animatronic still standing outside. + +The premise about the view was wrong, and nobody had checked it. With the door shut, the light still reaches the window, and that is exactly how a player tells whether the threat has gone. Bonnie, outside the left door, casts a shadow across the window that blocks the light, and the shadow disappears when he leaves. Chica stands in the right window and stays visible there for as long as she waits. It took a night of watching the fly reopen into a hallway it had not looked at to notice that the game had been offering the answer the whole time. + +The worry about the reference does not hold either. The live calibration runs at midnight, when all four animatronics are on stage, so a closed-door reference captured then is an empty window behind a shut door. + +Two findings from the Drosophila literature shape what replaced the blind reopening. Visual neurons in the fly receive motor-related input that predicts, and cancels, the image change a voluntary turn will cause (Kim, Fitzgerald and Maimon 2015), so comparing the eye against what the fly's own motor act should produce is a biological operation rather than a convenience. A repeated visual threat also leaves a persistent, slowly decaying defensive state whose length grows with the number of threats (Gibson et al. 2015), which is the role the hold trace already plays. ## Decision -The door is held shut by a leaky trace of the Giant Fiber drive, the same shape as `explore_drive` in ADR 0010. A spike charges `door_hold` to 1.0; the trace decays toward `release_threshold`, and when it falls under it the descending drive that was holding the door has stopped and the door relaxes open. With a leak of 0.972 per nominal frame and a threshold of 0.25 that is a hold of about 4.9 seconds. +The door is held shut by a leaky trace of the Giant Fiber drive, the same shape as `explore_drive` in ADR 0010. A spike charges `door_hold` to 1.0, and the trace decays toward `release_threshold` with a leak of 0.972 per nominal frame, about 4.9 seconds from a full charge. The decay is applied as `leak ** (elapsed * target_fps)` against the clock rather than once per iteration, because the first night run measured a mean hold of 7.5 seconds when the loop was really turning at 6.5 Hz. + +The live calibration records each hallway with the door open and with the door closed for a moment, after `motion_settle_sec` for the animation, and keeps the views it sees in a bank per door state (ADR 0001). + +The eye compares each hallway against the bank that matches the door's current state, which `DoorControl` publishes in `SensoryState.door_closed`. A shut door therefore reads as the expected closed view and drives nothing, while Bonnie's shadow or Chica in the window reads as a deviation and drives LPLC2 and LC4 exactly as a body in an open doorway does. The closed view has its own threshold, `closed_door_mse_threshold`, because a shadow is a fainter stimulus than a lit animatronic and has to be fitted on its own measurement. -The decay is applied as `leak ** (elapsed * target_fps)` against the wall clock rather than once per iteration. The first night run measured a mean hold of 7.5 seconds against the 4.9 the config asks for, because `engine.step` does not keep up with `target_fps` and the loop was really turning at about 6.5 Hz. Anchoring the decay to elapsed time makes the configured number mean what it says whatever the engine manages that night. +The eye is blind on that side only while a door command is in flight and for `motion_settle_sec` after the motor reports it done, so the moving door is not read as a threat. During a hold the fly looks at that side like any other. -A release is only acted on while the monitor is down. The door buttons are not on screen behind a raised tablet, so a click there does nothing, and firing it anyway would leave the controller believing the door is open while the game still holds it shut, a desync that never heals. The trace keeps decaying while the tablet is up; only the motor act waits. +A Giant Fiber spike on a side whose door is already shut renews the hold to 1.0 and is recorded as a renewal. A threat still standing in the window keeps its own door shut, through the same pathway that closed it. -While a door is shut, that side's visual input is gated to zero, and `state.blind_until[side]` is pushed forward every frame. This is the same corollary-discharge principle ADR 0006 already applies to the camera: the motor act that would saturate a sensory channel also suppresses it. It is also literally true, since a closed door blocks the view of the hallway behind it. The gate outlives the reopen by `reopen_settle_sec`, so the door's own opening animation is not read as motion. +Behind a shut door the eye reads the window and not the doorway. `VisionCalibration.left_window` and `right_window` are rectangles of their own, captured alongside the hallway boxes, and the closed-door bank holds views of that rectangle only. The bank keeps a single view per side, the brightest frame of the lit sweep, which is the empty window with the light falling through it. Dark frames do not enter it, whether they come from the light being off, from the moment before it comes on or from a flicker. -The fly therefore does not verify before reopening. It reopens blind, vision returns half a second later, and if the threat is still standing there LPLC2 drives the Giant Fiber again within one or two frames and the door slams a second time. Verification is not a separate mechanism; it is what the existing 10 Hz loop does as soon as it can see again. +The door reopens when two things are true at once. The hold has decayed under `release_threshold`, which is the defensive state running out, and the most recent look at that side since the last renewal came back clear. A look is clear when the light was confirmed on screen, the Giant Fiber stayed silent, the eye never drove the cluster, and the eye was not blind when the look began. Nothing is pressed while the tablet is up or the camera map is on screen, since the door buttons are not there to be clicked, and pressing anyway would leave the controller believing in a door the game never moved. ## Consequences -- Power is no longer spent indefinitely on a threat that walked away three hours ago, which is the difference between reaching 6 AM and browning out. -- An animatronic that waits produces a visible chatter: slam, hold ~5s, reopen, re-slam. That is an honest read of the animal, a transient escape reflex re-triggering, and it is the most legible behaviour the panel will show all night. -- The chatter is bounded by cost. Each door action carries `pan_delay_sec` of mouse travel before the click, and the motor queue is shared with lights and camera, so a much shorter hold would starve the other motor paths. `hold_leak_per_frame` is the knob if a night shows the fly toggling too often. -- The hold outlives `motor_refractory_sec`, checked in `tests/test_door_release.py`, so the Giant Fiber is always out of refractory by the time the door it closed comes back up and can immediately close it again. -- The fly is blind on a shut side. Nothing reads the hallway behind a closed door, which is correct, but it means a hold is a commitment: the fly cannot learn during those five seconds. -- `record_door_release` gives the session report the hold durations and the total door-closed time, which is the number that actually predicts whether the power lasts. +- The door opens because the fly saw the hallway empty, not because a trace ran out. A threat that waits keeps the door shut for as long as it is seen, and the session report counts each renewal. +- Nothing forces a look at the closed side, so a door can stay shut past the 4.9 second hold while the search drive looks elsewhere. The report prints that wait as the wait for the clearing look. The starvation guard still reaches every side within thirty seconds, so the wait is bounded, but it is power the night pays for. +- The chatter the first version accepted, reopening blind into a waiting animatronic and slamming again, is replaced by a hold that persists through the window, which also means fewer door actions competing for the motor queue. +- The first night after the change runs a live calibration and closes and opens each door once at midnight. Both hallways have to be empty for it, as they always are at that hour. +- The first live measurement of Bonnie behind the left door, on night 2, showed why the fly reopened into him. The left eye box spans x 0 to 480 of the game and is mostly door and buttons, while his shadow covers x 420 to 517 and y 263 to 541, so 42 percent of it fell outside the box and the rest was diluted into a distance of 16 against the empty window. Worse, the bank held a frame from the lit sweep in which the light had not yet reached the window, and inside the old box that dark window and Bonnie are the same picture. Twenty-five readings with him outside measured 2 against that frame, under a threshold of 40. +- With the light on and the window empty, the closed-door view is the same image every time and reads exactly 0. On the window rectangle, x 340 to 540 and y 200 to 560, Bonnie measures about 77 against it, so `closed_door_mse_threshold` is 30. The right window still uses the right eye box until Chica is measured the same way. +- A flicker behind a shut door now reads as a deviation, since no dark view is in the bank. The cost is a hold renewed for a moment longer, the direction the door can afford to be wrong in. +- A look taken while the light would not come on no longer counts as clear. It used to, because a dark window matched the dark views of the bank. +- The release still depends on a trace that is not part of the connectome. What changed is that the trace no longer decides on its own. | Version | Description | Author(s) | Date | Reviewer(s) | Review Date | |---|---|---|---|---|---| | 1.0 | Door reopening driven by the decay of the Giant Fiber trace | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-16 | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-16 | | 1.1 | Editorial pass for consistency with the rest of the documentation | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-18 | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-18 | +| 1.2 | Kept the eye open behind a closed door against a closed-door reference and made reopening wait for a clear look | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-10-01 | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-10-01 | +| 1.3 | Described the closed-door views as a bank and recorded the first live hold | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-10-01 | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-10-01 | +| 1.4 | Read the window rectangle behind a shut door against its lit view only, fitted the threshold on Bonnie's shadow and required a lit look to clear | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-10-01 | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-10-01 | diff --git a/docs/adr/0015-the-monitor-comes-down-only-when-the-office-is-back.md b/docs/adr/0015-the-monitor-comes-down-only-when-the-office-is-back.md index b9b808a..e12b459 100644 --- a/docs/adr/0015-the-monitor-comes-down-only-when-the-office-is-back.md +++ b/docs/adr/0015-the-monitor-comes-down-only-when-the-office-is-back.md @@ -1,42 +1,35 @@ # ADR 0015: The Monitor Comes Down Only When the Office Is Back on Screen ## Status -Accepted. Extends ADR 0014, which decided when the tablet rises but said nothing about how it is put away. +Accepted. Extends ADR 0014, which decided when the tablet rises but said nothing about how it is put away. Amended on 2026-10-01, when the office reference was replaced by the camera map as the evidence the tablet is up. ## Context -The 2026-09-17 night ended with the operator lowering the tablet by hand. The log explains why, and it is worth reading in order. +The 2026-09-17 night ended with the operator lowering the tablet by hand. At 17:50:47 DNp09 reached the bound and `open_camera` was issued in the same second as a right light check, two acts the game cannot run together because the office buttons sit behind the tablet. Six seconds later `close_camera` was issued and the loop marked the tablet as down without checking the screen. The gesture had missed, and one second later the office reference was recaptured on a timer, with the tablet in it. From there the fly read the tablet as the office and the office as the tablet, drove the inhibitors at full rate once the operator lowered it by hand, and every remaining light check came from the starvation guard. -At 17:50:47 DNp09 reached the bound and `open_camera` was issued. In the same second a right light check fired. Those two are mutually exclusive in the game, because the office buttons sit behind the tablet, so one of them was always going to be wasted. The engine loop evaluated `can_look` before it evaluated the camera, so a frame could queue a look and then raise the tablet underneath it, and the effector would run the look regardless. +The first fix closed the lowering into a loop and refused to recapture the office reference while the detector still read the tablet. On 2026-10-01 a night went wrong in the same way anyway. The tablet was on screen from the start, the fly believed it was down, it reached for lights and doors with the tablet up, the tablet appeared to open on Show Stage and never switch to CAM 1C, and a door reopening was clicked onto the tablet and never reached the game. The right eye had read the camera map as a threat in the first minute, and the inhibitors were not driven once in 682 frames. -Six seconds later `camera_watch_max_sec` expired, `close_camera` was issued, and the loop set `cam_is_open = False` without ever checking the screen. The gesture slides the cursor away from the tablet bar, which is not what FNAF listens for, so the tablet stayed up. One second after that, `_refresh_camera_closed_reference` recaptured the office reference on a blind timer and stored a picture of the tablet as the reference for "office visible". +Measuring the detector against the game explained it. Over eighteen frames of a live night, the centre patch read 377 to 893 MSE against the office reference with the tablet up on four different cameras, under the 1200 trigger, and 1359 with nothing more than the office turned to the right, over it. The detector was close to a coin toss, and once it misread a single lowering, the guard that protects the reference had nothing true to check against. `centre_view` did not help either, because FNAF scrolls the office for as long as the cursor sits outside a central band, so the reference was captured while the view was still moving and read 679 against itself. -From there everything inverts. `is_camera_up` compares the centre patch against that reference, so the fly read the tablet as the office and the office as the tablet. Once the operator lowered it by hand, `cam_inhib` was driven at the full sensory rate for the rest of the run. Per ADR 0006 that pins the GABAergic inhibitor clusters and zeroes the giant fiber, which is the same catatonia experiment 09 chased down in a different disguise. After 17:51:58 every light check in the log carries the reason `guard`. The brain had stopped contributing. - -The detector helped none of this. `is_camera_up` read only the newest frame in the buffer, so a single frame of the office fan or the animated poster could trip a 100 MSE threshold on its own. +The gestures were measured on the same night. A cursor teleported onto the bar raised the tablet, but the same teleport did not lower it, and the slide that `close_camera` used crosses the bar in about 16 ms and was ignored on one of two tries. A slide down onto the bar that ends with a few small moves on it flipped the tablet eight times out of eight, alternating, with the map gone in under 0.1 s on the way down and visible after 0.2 to 0.3 s on the way up. ## Decision -Lowering the tablet becomes a closed loop that ends on evidence rather than on a click having been sent. - -`MonitorControl` owns the whole tablet state and is the only thing that writes `state.camera_open`. When the drive fades or the power cap expires it starts `_lower_monitor`, which issues the slide, waits out `close_settle_sec`, and asks the screen. If the office is back it re-anchors the reference and reports success. If the office is not back it issues `nudge_camera_bar`, a gesture that drives the cursor onto the bar and off again, and asks once more. Only after both gestures have failed does it warn, and it warns with the measured MSE so the operator can tell a stuck tablet from a stale reference. The fly keeps believing the tablet is up, retries every `lower_retry_sec`, and recovers on its own the moment the tablet actually comes down, whoever lowered it. - -`_reanchor_office_reference` refuses to capture while the detector still reads the tablet. The reference can therefore drift slowly with the office as the night wears on, which is what experiment 09 wanted, but it can never flip to a picture of the tablet. That single guard is what breaks the cascade above. +The fly decides whether the tablet is up from the camera map, not from a picture of the office. The map region, x 820 to 1240 and y 345 to 680 in the 1280x720 reference layout, is captured with the eye patches, scaled back to that layout, and thresholded at a grey level of 200. Every closed white contour the size of a camera button, 40 to 75 by 25 to 45 pixels and at least 60 percent filled, counts once, and a contour inside one already counted does not. The FNAF 1 map has eleven buttons. The tablet is up when the last three frames all show at least six of them and down when all three show fewer, and in between the fly does not know. On the live frames this count read eleven in all eleven tablet frames and zero in all seven office frames, whichever way the office faced. Nothing is captured as a reference, so there is nothing that can be poisoned. -The two motor programs are made mutually exclusive in the loop rather than by luck. The camera decision now runs first, `can_look` requires the tablet to be down, and the tablet is not allowed to rise while a look is queued or running. `_saccade_task` also waits for the office before it touches a light button, because the request and the effector are separate coroutines and a frame boundary sits between them. +There is one gesture, `flip_tablet`, a slide from y 540 down onto the bar followed by four small moves on it, and the controller keeps no opinion about the tablet. `put_tablet` checks the screen first and does nothing if the tablet is already where it is wanted, then flips and waits up to `flip_confirm_sec` for the map to agree. Raising works the same way, so the fly only believes the tablet is up once the map is on screen, and a raise the game ignored is reported and dropped. A lowering that is not confirmed is retried every `lower_retry_sec`, and lowering it by hand works because the screen is what the fly believes. -Competition between the two stays where ADR 0014 put it. A search drive above `camera_release_forage_bias` is one of the ways watching ends, so a fly that badly wants to check a hallway puts the tablet down first instead of reaching through it. +A camera map on screen while the fly believes the tablet is down is put away, and the report counts it. The inhibitors follow the map rather than the fly's belief, light checks wait until no map is on screen, and the door logic presses nothing while it is there. A night therefore starts by making sure the tablet is down, before any light or door is touched. -`is_camera_up` now takes the minimum MSE across the whole capture buffer, the same way the hallway detectors already did. A transient animation frame no longer reads as a raised tablet, because every recent frame has to disagree with the reference before the fly believes it. +The two motor programs stay mutually exclusive as before. The camera decision runs first, a look cannot be requested while a raise or a lowering is in flight, and the tablet does not rise while a look is queued or running. ## Consequences -- The fly can no longer be left blind by a click that did not land, and the run says so in the log and in the session report under `Lowerings That Needed Retry`. -- A failed lowering costs roughly two seconds before the retry, against a night that was previously lost outright. -- `MotorRefrac` loses its `camera` field and the camera refractory moves into `MonitorControl`, which is the only thing that knows when the tablet really came down. -- Light checks and camera pulls can no longer collide, at the cost of one frame of lag on `state.forage_bias`, which the monitor reads from the previous frame to avoid a circular dependency inside the loop. -- `close_camera` accepts `force`, because the worker's own `camera_state` guard would otherwise swallow every retry. -- Detection is now roughly 80ms slower to notice the tablet going up, which is the buffer depth, and considerably less likely to be wrong. -- The gesture question is still open. It is not known which of the two gestures FNAF actually responds to, only that the fly now finds out on its own and reports what happened. +- A single missed gesture no longer inverts the night. The worst case is a raise that is dropped or a lowering that takes another three seconds, and the report shows both under the tablet gestures missed. +- The capture loop counts contours on a 420 by 335 grey image every pass, which is cheap next to the screen grabs it already makes. +- The detector depends on the FNAF 1 map drawing eleven white-bordered buttons at a fixed place on screen. A different game version or a mod that moves the map would need the region and the button size measured again, and `debug_camera.py` prints the live count for that. +- `centre_view`, `nudge_camera_bar`, the office reference and its recapture are gone, and so is the warning about a stale office reference, which described a failure that can no longer happen. +- The DNp04 escape drops the tablet, waits for the map to leave the screen and only then closes the door, so a door click can no longer land on the tablet. | Version | Description | Author(s) | Date | Reviewer(s) | Review Date | |---|---|---|---|---|---| | 1.0 | Tablet lowering closed into a loop and the office reference protected | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-17 | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-17 | | 1.1 | Editorial pass for consistency with the rest of the documentation | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-18 | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-18 | +| 1.2 | Replaced the office reference with the camera map as the evidence for the tablet, and a single gesture confirmed on screen | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-10-01 | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-10-01 | diff --git a/docs/adr/0024-the-night-is-a-package-and-main-only-wires-it.md b/docs/adr/0024-the-night-is-a-package-and-main-only-wires-it.md index 9abd5c6..03e9c1a 100644 --- a/docs/adr/0024-the-night-is-a-package-and-main-only-wires-it.md +++ b/docs/adr/0024-the-night-is-a-package-and-main-only-wires-it.md @@ -24,7 +24,7 @@ and `main.py` keeps only the wiring of a run and its entry function. | `night/engine.py` | `ConnectomeEngine` | | `night/motor.py` | `await_motor`, the wait on a motor command | | `night/calibration.py` | the countdown, the game in front check, the eye reference capture and `calibrate` | -| `night/monitor.py` | `MonitorControl` and the recovery of the office reference | +| `night/monitor.py` | `MonitorControl` and the tablet gesture confirmed on screen | | `night/tasks.py` | the vision, saccade and engine loops and `observe_hallway` | | `night/panel.py` | the browser window of the panel, its pinning, its focus handback and its shutdown | @@ -64,3 +64,4 @@ print it, so nothing visible changed. | Version | Description | Author(s) | Date | Reviewer(s) | Review Date | |---|---|---|---|---|---| | 1.0 | Recorded the split of main into the night package | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-30 | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-30 | +| 1.1 | Updated the monitor row after the office reference was retired | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-10-01 | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-10-01 | diff --git a/docs/adr/0026-the-tablet-feeds-the-lobula-and-dnp04-answers-through-it.md b/docs/adr/0026-the-tablet-feeds-the-lobula-and-dnp04-answers-through-it.md index 93bbaf9..ad440c4 100644 --- a/docs/adr/0026-the-tablet-feeds-the-lobula-and-dnp04-answers-through-it.md +++ b/docs/adr/0026-the-tablet-feeds-the-lobula-and-dnp04-answers-through-it.md @@ -26,7 +26,8 @@ The tablet reads the west side through two lobula channels, chosen by the camera - The cove is compared against a bank of frames captured at the start of the night with the curtain closed. Taking the nearest frame in the bank absorbs the camera pan. The distance above the bank's own noise, over `figure_span_mse`, drives LC9 and LC31a on the left. - A DNp09 spike while the cove is readable moves the gaze to CAM 2A, once per raise. The object has left and the fly follows it. - In CAM 2A the same distance drives LPLC2 and its growth per second drives LC4, the size and velocity split of the door reflex. -- A DNp04 spike while the tablet is up drops the tablet, closes the door on that side, recentres the view and re-anchors the office before the lowering counts as done. +- A DNp04 spike while the tablet is up drops the tablet, waits for the camera map to leave the screen and only then closes the door on that side (ADR 0015). +- A watch only counts as boring once the feed has been readable for `camera_watch_min_sec`. Counted from the raise, as it first was, every watch on 2026-10-01 ended 0.7 to 1.0 seconds in, before the 1.2 second settle had passed, and CAM 1C never reached the brain once in a night Foxy won. Counted from the moment the feed is readable, the same night's watches fed 0.02 to 0.27 of figure drive while Foxy changed in the cove. That sits around the 0.25 the table above needs for DNp09, so a peeking Foxy is noticed weakly and rarely pursued, and the empty cove with its sign has not yet been caught on a watch. `env/tablet_feed.py` captures and compares the feed, `night/tablet/` holds the transduction, the gaze, the per-frame watch and the start-of-night references, and the door bookkeeping moves out of the engine loop into `night/doors.py` so the Giant Fiber and DNp04 close doors through one path. @@ -42,3 +43,5 @@ The tablet reads the west side through two lobula channels, chosen by the camera | Version | Description | Author(s) | Date | Reviewer(s) | Review Date | |---|---|---|---|---|---| | 1.0 | CAM 1C drives LC9 and LC31a, DNp09 pursues to CAM 2A, CAM 2A drives LPLC2 and LC4, DNp04 closes the door | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-30 | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-30 | +| 1.1 | Described the escape against the camera map instead of the office reference | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-10-01 | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-10-01 | +| 1.2 | Counted the minimum watch from the moment the feed is readable and recorded the drive Foxy produced | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-10-01 | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-10-01 | diff --git a/docs/experiments/14-the-shadow-that-looked-like-the-dark.md b/docs/experiments/14-the-shadow-that-looked-like-the-dark.md new file mode 100644 index 0000000..0804140 --- /dev/null +++ b/docs/experiments/14-the-shadow-that-looked-like-the-dark.md @@ -0,0 +1,41 @@ +# Experiment 14: The Shadow That Looked Like The Dark + +The doors finally reopened on their own (ADR 0011), and the first thing the fly did with that was reopen the left door with Bonnie standing right behind it. His shadow was in the window, I could see it, and the fly looked at the window with the light on and decided the hallway was empty. Then it opened the door and died (Bonnie was very happy about it). + +The closed-door threshold was 40, copied from the open door, and ADR 0011 already admitted it had never been fitted on a shadow. So the plan was simple, measure the window with the door closed, once empty and once with Bonnie, and put the threshold in between. + +### Measuring It Badly, Twice + +The empty reading came out at exactly 0 on every frame. FNAF draws the lit window behind a closed door from one pre-rendered image, so an empty window is pixel for pixel the reference. Good start. + +Then I left a script closing the door, turning the light on every couple of seconds and waiting for Bonnie. It read 371 over and over, which looked like a lot of Bonnie for a hallway with nobody in it. The door was open. The controller toggles each door from what it believes, the previous script had left the door shut, and the new process started believing it was open, therefore the click meant to close it opened it. I had been measuring an open hallway against closed-door references for minutes (lol). + +The second attempt checked the door button colour on screen before every click, which is the right idea, except it sampled a fixed point and the button is only there when the office is turned to the left. It read black, decided nothing was where it should be and kept clicking. At that point I stopped letting a script touch the game at all. I played the night myself, handled Foxy and the right door, and a second script only watched the screen and saved every frame while the left light was on. + +### Bonnie Measured 2 + +When Bonnie came, I closed the door and held the light on his shadow. Twenty-five readings, all of them 2 against the closed-door bank, under a threshold of 40. For the eye he basically did not exist. + +My first guess was the box. The left eye reads a 481 pixel square from x 0 to 480 of the game, and most of that square is the door and its buttons. Bonnie's shadow sits from x 420 to 517 and y 263 to 541, so 42 percent of it falls outside the box, and the part inside is diluted by everything that does not change. Against the empty window, the shadow inside the box measured 16. That explains why 40 could never catch him, but it does not explain 2. + +My second guess was that the calibration had recorded Bonnie. The bank had three closed-door views and the one he matched at 2, `left_closed_01`, did look different from the empty one in exactly the same place. That turned out to be wrong as well. Inside the window, that view was identical to the view recorded with the light off, MSE 0.0. It was not Bonnie, it was a frame from the lit sweep taken before the light had actually reached the window. + +### What The Light Actually Adds + +Putting the frames side by side made it obvious. With the door closed, the hallway light adds almost nothing to the window, just a thin vertical streak of light around x 450. Everything else in the window is the office itself, the same with the light on or off. When Bonnie stands there, his body covers that streak, and his head shows up as a dark shape a bit further right, mostly past the edge of the eye box. + +So inside the old box, a dark window and Bonnie are the same picture. The bank held a dark window as a normal view, and the comparison picks the nearest view, therefore Bonnie read as normal. The same hole existed from another side too. A look where the light would not come on still counted as a clear look, because a dark window matched the dark views of the bank. + +### What Changed + +Behind a closed door the eye now reads the window itself, a rectangle from x 340 to 540 and y 200 to 560 captured on its own, with the whole shadow inside and the door left out. The closed-door bank keeps only the brightest frame of the lit sweep, which is the empty window with the streak in it, and no dark view enters it anymore. On that rectangle Bonnie measures about 77 against the empty window, so the threshold went to 30. A look only clears a door when the light was confirmed on screen. If the light flickers behind a closed door, the fly reads a threat and holds the door a little longer, and I would rather pay that in power than in jumpscares. + +### The Night After + +The first night with the change recalibrated (the layout had changed) and then did exactly what it should. At 20:59:16 Bonnie showed up in the open doorway at 118 and the giant fiber closed the left door. Thirty seconds later the fly looked at the window behind the closed door, read 95 against 30 and the giant fiber fired again, so the hold was renewed instead of the door opening into him. At 21:00:02 it looked again, read 0, and only then released the door, after 46.5 seconds. Chica on the right read 482 behind the closed right door and kept it shut the same way. + +It got close to the end of night 3 and then lost it on the left. At 21:01:12 a look read 118 with the door open, which is Bonnie in the doorway, but the eye was driven on only 1 of 12 frames and the giant fiber never answered, so the door stayed open. Twenty seconds later the left light would not come on, which in FNAF 1 is what happens when an animatronic is already inside the office. I think the light did not jam on its own, the fly missed him by one frame first. Why a doorway reading of 118 drove the eye only once is the next thing to measure, as well as the right window, which still uses the old box until I measure Chica the same way. + +| Version | Description | Author(s) | Date | Reviewer(s) | Review Date | +|---|---|---|---|---|---| +| 1.0 | Why the fly reopened into Bonnie, the dark window hiding his shadow, the window rectangle and the first night with it | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-10-01 | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-10-01 | diff --git a/src/flynaf/config.py b/src/flynaf/config.py index 4c4ec42..415cb10 100644 --- a/src/flynaf/config.py +++ b/src/flynaf/config.py @@ -22,6 +22,8 @@ class VisionCalibration: right_target_x: int = 851 right_target_y: int = 328 right_bbox_size: int = 335 + left_window: tuple = (340, 200, 540, 560) + right_window: tuple = (684, 161, 1019, 496) @dataclass(frozen=True) @@ -36,8 +38,7 @@ class MotorCalibration: right_light_button_y: int = 467 camera_hover_x: int = 552 camera_hover_y: int = 665 - camera_bar_dwell_sec: float = 0.25 - screen_center_y: int = 540 + camera_bar_approach_y: int = 540 click_delay_sec: float = 0.05 pan_delay_sec: float = 1.0 @@ -56,7 +57,7 @@ class SimulationParams: @dataclass(frozen=True) class ForagingParams: subliminal_noise_hz: float = 1.0 - mse_threshold: float = 1500.0 + mse_threshold: float = 40.0 light_inspection_frames: int = 12 light_inspection_max_sec: float = 8.0 light_activation_settle_sec: float = 0.4 @@ -66,15 +67,18 @@ class ForagingParams: camera_watch_max_sec: float = 6.0 camera_watch_min_sec: float = 0.8 camera_release_forage_bias: float = 0.5 - + closed_door_mse_threshold: float = 30.0 @dataclass(frozen=True) class VisionDynamics: frame_buffer_size: int = 5 capture_delay_sec: float = 0.016 - calibration_peak_frames: int = 10 - calibration_peak_delay_sec: float = 0.03 + reference_sweep_sec: float = 4.0 + bank_tolerance_mse: float = 5.0 + light_button_half_size: int = 10 + light_button_lit_level: float = 140.0 + light_check_interval_sec: float = 0.08 @dataclass(frozen=True) @@ -94,14 +98,18 @@ class NeuralParams: @dataclass(frozen=True) class CameraDetection: - patch_x: int = 521 - patch_y: int = 300 - patch_size: int = 200 - mse_trigger: float = 1200.0 - close_settle_sec: float = 1.0 - stuck_warn_sec: float = 5.0 + map_left: int = 820 + map_top: int = 345 + map_right: int = 1240 + map_bottom: int = 680 + white_level: int = 200 + button_width: tuple = (40, 75) + button_height: tuple = (25, 45) + button_fill: float = 0.6 + min_buttons: int = 6 + agreeing_frames: int = 3 + flip_confirm_sec: float = 1.5 lower_retry_sec: float = 3.0 - lower_confirm_sec: float = 3.0 lower_gesture_attempts: int = 4 @@ -173,7 +181,7 @@ class SearchDynamics: class DoorDynamics: hold_leak_per_frame: float = 0.972 release_threshold: float = 0.25 - reopen_settle_sec: float = 0.5 + motion_settle_sec: float = 0.5 @dataclass(frozen=True) diff --git a/src/flynaf/env/brain_view.py b/src/flynaf/env/brain_view.py index 2bf743a..0aa2913 100644 --- a/src/flynaf/env/brain_view.py +++ b/src/flynaf/env/brain_view.py @@ -1,6 +1,7 @@ import asyncio import base64 import csv +import dataclasses import json import os import struct @@ -31,9 +32,35 @@ } +@dataclasses.dataclass(frozen=True) +class PanelFrame: + gaze: str = '--' + camera: bool = False + cam: str | None = None + gf_l: bool = False + gf_r: bool = False + in_l: bool = False + in_r: bool = False + look_l: bool = False + look_r: bool = False + mse_l: float | None = None + mse_r: float | None = None + mse_th: float = 0.0 + eye_l: int = 0 + eye_r: int = 0 + door_l: bool = False + door_r: bool = False + figure: float = 0.0 + loom: float = 0.0 + dnp09: bool = False + dnp04_l: bool = False + dnp04_r: bool = False + + class SpikeFeed: def __init__(self, num_neurons: int = 0, tracer=None): self.indices = np.empty(0, dtype=np.uint32) + self.panel = PanelFrame() self._seen = np.zeros(num_neurons, dtype=bool) self._tracer = tracer self.fired_total = 0 @@ -42,7 +69,9 @@ def __init__(self, num_neurons: int = 0, tracer=None): self.cascade = None self.cascade_revision = 0 - def publish(self, spiking: np.ndarray): + def publish(self, spiking: np.ndarray, panel: PanelFrame | None = None): + if panel is not None: + self.panel = panel self.indices = np.unique(spiking).astype(np.uint32, copy=False) self.spike_total += int(self.indices.size) if self._seen.size: @@ -181,9 +210,8 @@ def build_blobs() -> tuple: class BrainViewServer: - def __init__(self, feed: SpikeFeed, highlights: dict, eyes=None): + def __init__(self, feed: SpikeFeed, eyes=None): self._feed = feed - self._highlights = highlights self._eyes = eyes self._geometry = None self._outline = None @@ -196,8 +224,7 @@ def page_height(self) -> int: return self._page_height async def start(self) -> int: - loop = asyncio.get_event_loop() - self._geometry, self._outline = await loop.run_in_executor(None, build_blobs) + self._geometry, self._outline = await asyncio.to_thread(build_blobs) self._server = await asyncio.start_server( self._handle, '127.0.0.1', self._params.port ) @@ -260,7 +287,7 @@ async def _stream(self, writer: asyncio.StreamWriter): feed = self._feed if feed.revision != sent: sent = feed.revision - frame = dict(self._highlights) + frame = dataclasses.asdict(feed.panel) frame['s'] = base64.b64encode(feed.indices.tobytes()).decode('ascii') frame['n'] = feed.revision frame['fired'] = feed.fired_total diff --git a/src/flynaf/env/hallway_bank.py b/src/flynaf/env/hallway_bank.py new file mode 100644 index 0000000..8aae887 --- /dev/null +++ b/src/flynaf/env/hallway_bank.py @@ -0,0 +1,22 @@ +import numpy as np + + +def frame_distance(frame: np.ndarray, reference: np.ndarray) -> float: + return float(np.mean((frame - reference) ** 2)) + + +def nearest(frame: np.ndarray, bank: list) -> tuple: + distances = [frame_distance(frame, reference) for reference in bank] + index = int(np.argmin(distances)) + return distances[index], bank[index] + + +def add_if_new(bank: list, frame: np.ndarray, tolerance: float) -> bool: + if bank and nearest(frame, bank)[0] <= tolerance: + return False + bank.append(frame) + return True + + +def median_index(values: list) -> int: + return int(np.argsort(values)[len(values) // 2]) diff --git a/src/flynaf/env/input_controller.py b/src/flynaf/env/input_controller.py index ca57f92..f10e64a 100644 --- a/src/flynaf/env/input_controller.py +++ b/src/flynaf/env/input_controller.py @@ -49,21 +49,18 @@ def _click(): time.sleep(config.MOTOR_CALIBRATION.click_delay_sec) desktop.mouse_up() - def _slide(x: int, y_from: int, y_to: int, steps: int = 20, delay: float = 0.008): - side = _pan_side(x) - swinging = side != office_facing() - _set_facing('panning') + def _flip(x: int, y_from: int, y_to: int, steps: int = 20, delay: float = 0.008): + desktop.move_cursor(*anchor.point(x, y_from)) for i in range(1, steps + 1): y = y_from + int((y_to - y_from) * i / steps) desktop.move_cursor(*anchor.point(x, y)) time.sleep(delay) - if swinging: - time.sleep(config.MOTOR_CALIBRATION.pan_delay_sec) - _set_facing(side) - + for offset in (8, 16, 8, 0): + time.sleep(0.04) + desktop.move_cursor(*anchor.point(x + offset, y_to)) + _set_facing(_pan_side(x)) light_state = {'left': False, 'right': False} - camera_state = {'open': False} door_state = {'left': False, 'right': False} while True: @@ -71,14 +68,14 @@ def _slide(x: int, y_from: int, y_to: int, steps: int = 20, delay: float = 0.008 action = cmd.get('action') done = cmd.get('done') try: - _dispatch(cmd, action, _reach, _click, _slide, light_state, camera_state, door_state) + _dispatch(cmd, action, _reach, _click, _flip, light_state, door_state) finally: if done is not None: done.set() _CMD_QUEUE.task_done() -def _dispatch(cmd, action, _reach, _click, _slide, light_state, camera_state, door_state): +def _dispatch(cmd, action, _reach, _click, _flip, light_state, door_state): motor = config.MOTOR_CALIBRATION if action in ('close_left_door', 'open_left_door'): @@ -109,18 +106,14 @@ def _dispatch(cmd, action, _reach, _click, _slide, light_state, camera_state, do _click() light_state['right'] = state - elif action == 'centre_view': - _reach(motor.camera_hover_x, motor.screen_center_y) + elif action in ('press_left_light', 'press_right_light'): + side = action.split('_')[1] + _reach(getattr(motor, f'{side}_light_button_x'), getattr(motor, f'{side}_light_button_y')) + _click() + light_state[side] = cmd.get('state', not light_state[side]) - elif action == 'open_camera': - if not camera_state['open']: - _slide(motor.camera_hover_x, motor.screen_center_y, motor.camera_hover_y) - camera_state['open'] = True - - elif action == 'close_camera': - if camera_state['open'] or cmd.get('force'): - _slide(motor.camera_hover_x, motor.camera_hover_y, motor.screen_center_y) - camera_state['open'] = False + elif action == 'flip_tablet': + _flip(motor.camera_hover_x, motor.camera_bar_approach_y, motor.camera_hover_y) elif action == 'select_camera': button = _camera_button(cmd.get('camera')) @@ -130,12 +123,6 @@ def _dispatch(cmd, action, _reach, _click, _slide, light_state, camera_state, do time.sleep(motor.click_delay_sec) _click() - elif action == 'nudge_camera_bar': - _slide(motor.camera_hover_x, motor.screen_center_y, motor.camera_hover_y) - time.sleep(motor.camera_bar_dwell_sec) - _slide(motor.camera_hover_x, motor.camera_hover_y, motor.screen_center_y) - camera_state['open'] = False - else: _log.warning('unknown command action: %s', action) @@ -182,17 +169,14 @@ def set_left_light(self, state: bool) -> threading.Event: def set_right_light(self, state: bool) -> threading.Event: return _submit(action='set_right_light', state=state) - def centre_view(self) -> threading.Event: - return _submit(action='centre_view') - - def open_camera(self) -> threading.Event: - return _submit(action='open_camera') + def press_left_light(self, state: bool) -> threading.Event: + return _submit(action='press_left_light', state=state) - def close_camera(self, force: bool = False) -> threading.Event: - return _submit(action='close_camera', force=force) + def press_right_light(self, state: bool) -> threading.Event: + return _submit(action='press_right_light', state=state) - def nudge_camera_bar(self) -> threading.Event: - return _submit(action='nudge_camera_bar') + def flip_tablet(self) -> threading.Event: + return _submit(action='flip_tablet') def select_camera(self, camera: str, settle_sec: float = 0.0) -> threading.Event: return _submit(action='select_camera', camera=camera, settle_sec=settle_sec) diff --git a/src/flynaf/env/overlay.py b/src/flynaf/env/overlay.py index c630c62..95eff55 100644 --- a/src/flynaf/env/overlay.py +++ b/src/flynaf/env/overlay.py @@ -25,13 +25,14 @@ def capture_regions() -> list: camera = config.CAMERA_DETECTION left = anchor.point(vision.left_target_x, vision.left_target_y) right = anchor.point(vision.right_target_x, vision.right_target_y) - patch = anchor.point(camera.patch_x, camera.patch_y) + map_corner = anchor.point(camera.map_left, camera.map_top) + map_far_corner = anchor.point(camera.map_right, camera.map_bottom) tablet = config.TABLET_VISION feed = anchor.point(tablet.feed_x, tablet.feed_y) return [ centered_rect(left[0], left[1], anchor.size(vision.left_bbox_size)), centered_rect(right[0], right[1], anchor.size(vision.right_bbox_size)), - centered_rect(patch[0], patch[1], anchor.size(camera.patch_size)), + (*map_corner, *map_far_corner), centered_rect(feed[0], feed[1], anchor.size(tablet.feed_size)), ] @@ -44,7 +45,7 @@ def motor_regions(pad: int = 40) -> list: (motor.right_door_button_x, motor.right_door_button_y), (motor.right_light_button_x, motor.right_light_button_y), (motor.camera_hover_x, motor.camera_hover_y), - (motor.camera_hover_x, motor.screen_center_y), + (motor.camera_hover_x, motor.camera_bar_approach_y), ] + [(button.x, button.y) for button in config.TABLET_VISION.cameras] reach = anchor.size(pad * 2) return [centered_rect(*anchor.point(x, y), reach) for x, y in targets] diff --git a/src/flynaf/env/tablet_map.py b/src/flynaf/env/tablet_map.py new file mode 100644 index 0000000..1f6a5bd --- /dev/null +++ b/src/flynaf/env/tablet_map.py @@ -0,0 +1,48 @@ +import cv2 +import numpy as np + +from flynaf import config +from flynaf.env import anchor + + +def design_size(params=None) -> tuple: + params = params or config.CAMERA_DETECTION + return (params.map_right - params.map_left, params.map_bottom - params.map_top) + + +def map_box() -> dict: + params = config.CAMERA_DETECTION + left, top = anchor.point(params.map_left, params.map_top) + right, bottom = anchor.point(params.map_right, params.map_bottom) + return {'left': left, 'top': top, 'width': max(1, right - left), 'height': max(1, bottom - top)} + + +def to_design(frame: np.ndarray, params=None) -> np.ndarray: + gray = frame if frame.ndim == 2 else cv2.cvtColor(frame, cv2.COLOR_BGRA2GRAY) + return cv2.resize(gray, design_size(params), interpolation=cv2.INTER_AREA) + + +def count_camera_buttons(gray: np.ndarray, params=None) -> int: + params = params or config.CAMERA_DETECTION + white = (gray > params.white_level).astype(np.uint8) + contours, _ = cv2.findContours(white, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE) + low_w, high_w = params.button_width + low_h, high_h = params.button_height + shaped = [] + for contour in contours: + x, y, width, height = cv2.boundingRect(contour) + if not (low_w <= width <= high_w and low_h <= height <= high_h): + continue + if cv2.contourArea(contour) > params.button_fill * width * height: + shaped.append((x, y, width, height)) + return len(distinct_buttons(shaped)) + + +def distinct_buttons(rects: list) -> list: + kept = [] + for x, y, width, height in sorted(rects, key=lambda rect: rect[2] * rect[3], reverse=True): + centre_x, centre_y = x + width / 2, y + height / 2 + if any(kx <= centre_x <= kx + kw and ky <= centre_y <= ky + kh for kx, ky, kw, kh in kept): + continue + kept.append((x, y, width, height)) + return kept diff --git a/src/flynaf/env/vision.py b/src/flynaf/env/vision.py index 8c9c42d..8ba18f0 100644 --- a/src/flynaf/env/vision.py +++ b/src/flynaf/env/vision.py @@ -1,4 +1,5 @@ import collections +import json import logging import os import threading @@ -9,14 +10,17 @@ import numpy as np from flynaf import config -from flynaf.env import anchor, desktop +from flynaf.env import anchor, desktop, hallway_bank, tablet_map _log = logging.getLogger(__name__) +SIDES = ('left', 'right') + class FNAFVision: def __init__(self): self.mse_threshold = config.FORAGING_PARAMS.mse_threshold + self.closed_mse_threshold = config.FORAGING_PARAMS.closed_door_mse_threshold self.l_x = config.VISION_CALIBRATION.left_target_x self.l_y = config.VISION_CALIBRATION.left_target_y @@ -26,18 +30,20 @@ def __init__(self): self.r_y = config.VISION_CALIBRATION.right_target_y self.r_bbox = config.VISION_CALIBRATION.right_bbox_size - self._cam_x = config.CAMERA_DETECTION.patch_x - self._cam_y = config.CAMERA_DETECTION.patch_y - self._cam_size = config.CAMERA_DETECTION.patch_size - self._cam_mse_trigger = config.CAMERA_DETECTION.mse_trigger + self.windows = { + 'left': config.VISION_CALIBRATION.left_window, + 'right': config.VISION_CALIBRATION.right_window, + } + + self._min_buttons = config.CAMERA_DETECTION.min_buttons - self._calib_frames = config.VISION_DYNAMICS.calibration_peak_frames - self._calib_delay = config.VISION_DYNAMICS.calibration_peak_delay_sec self._buf_size = config.VISION_DYNAMICS.frame_buffer_size + self._sweep_sec = config.VISION_DYNAMICS.reference_sweep_sec + self._bank_tolerance = config.VISION_DYNAMICS.bank_tolerance_mse + self._capture_delay = config.VISION_DYNAMICS.capture_delay_sec - self.ref_left = None - self.ref_right = None - self._ref_camera_closed = None + self.banks = {(side, closed): [] for side in SIDES for closed in (False, True)} + self._distance_cache = {} self.debug_dir = os.path.join(config.PROJECT_ROOT, 'logs', 'vision_debug') os.makedirs(self.debug_dir, exist_ok=True) @@ -47,7 +53,10 @@ def __init__(self): self._left_buf: collections.deque = collections.deque(maxlen=self._buf_size) self._right_buf: collections.deque = collections.deque(maxlen=self._buf_size) - self._cam_buf: collections.deque = collections.deque(maxlen=1) + self._window_bufs = {side: collections.deque(maxlen=self._buf_size) for side in SIDES} + self._map_buttons: collections.deque = collections.deque( + maxlen=config.CAMERA_DETECTION.agreeing_frames, + ) self._buf_lock = threading.Lock() self._threat_written_left = False self._threat_written_right = False @@ -74,18 +83,23 @@ def _grab_gray(self, sct, x: int, y: int, size: int) -> np.ndarray: centre_x, centre_y = anchor.point(x, y) size = anchor.size(size) offset = size // 2 - left = centre_x - offset - top = centre_y - offset + return self._grab_box(sct, centre_x - offset, centre_y - offset, size, size) + + def _grab_window(self, sct, side: str) -> np.ndarray: + left, top, right, bottom = self.windows[side] + screen_left, screen_top = anchor.point(left, top) + return self._grab_box(sct, screen_left, screen_top, + anchor.size(right - left), anchor.size(bottom - top)) + + def _grab_box(self, sct, left: int, top: int, width: int, height: int) -> np.ndarray: screen_w, screen_h = self._screen_size() if screen_w > 0 and screen_h > 0: - left = max(0, min(left, screen_w - size)) - top = max(0, min(top, screen_h - size)) + left = max(0, min(left, screen_w - width)) + top = max(0, min(top, screen_h - height)) else: left = max(0, left) top = max(0, top) - bbox = {'top': top, 'left': left, 'width': size, 'height': size} - img = sct.grab(bbox) - frame = np.array(img) + frame = np.array(sct.grab({'top': top, 'left': left, 'width': width, 'height': height})) return cv2.cvtColor(frame, cv2.COLOR_BGRA2GRAY).astype(np.float32) def _capture_loop(self): @@ -94,22 +108,47 @@ def _capture_loop(self): while True: left_frame = self._grab_gray(sct, self.l_x, self.l_y, self.l_bbox) right_frame = self._grab_gray(sct, self.r_x, self.r_y, self.r_bbox) - cam_frame = self._grab_gray(sct, self._cam_x, self._cam_y, self._cam_size) + windows = {side: self._grab_window(sct, side) for side in SIDES} + buttons = tablet_map.count_camera_buttons( + tablet_map.to_design(np.array(sct.grab(tablet_map.map_box()))), + ) with self._buf_lock: self._left_buf.append(left_frame) self._right_buf.append(right_frame) - self._cam_buf.append(cam_frame) + for side, frame in windows.items(): + self._window_bufs[side].append(frame) + self._map_buttons.append(buttons) time.sleep(delay) - def capture_left_reference(self): - self.ref_left = self._capture_peak_reference(self._left_buf) - - def capture_right_reference(self): - self.ref_right = self._capture_peak_reference(self._right_buf) + def _buffer(self, side: str, door_closed: bool) -> collections.deque: + if door_closed: + return self._window_bufs[side] + return self._left_buf if side == 'left' else self._right_buf + + def capture_bank(self, side: str, door_closed: bool, seconds: float | None = None, + brightest_only: bool = False) -> int: + bank = self.banks[(side, door_closed)] + buf = self._buffer(side, door_closed) + deadline = time.perf_counter() + (self._sweep_sec if seconds is None else seconds) + added, last, brightest = 0, None, None + while time.perf_counter() < deadline: + with self._buf_lock: + frame = buf[-1] if buf else None + if frame is not None and frame is not last: + last = frame + if not brightest_only: + added += hallway_bank.add_if_new(bank, frame, self._bank_tolerance) + elif brightest is None or frame.mean() > brightest.mean(): + brightest = frame + time.sleep(self._capture_delay) + if brightest is not None: + added += hallway_bank.add_if_new(bank, brightest, self._bank_tolerance) + self._distance_cache.clear() + return added def _layout_fingerprint(self) -> str: return '|'.join(str(getattr(self, name, None)) for name in ( - 'l_x', 'l_y', 'l_bbox', 'r_x', 'r_y', 'r_bbox', + 'l_x', 'l_y', 'l_bbox', 'r_x', 'r_y', 'r_bbox', 'windows', )) + '|' + str(anchor.game_rect()) def _cached_layout_matches(self) -> bool: @@ -122,31 +161,53 @@ def _cached_layout_matches(self) -> bool: _log.info('the cached eye references were taken on a different layout, recapturing') return False + @property + def _bank_index_path(self) -> str: + return os.path.join(self.reference_dir, 'banks.json') + def load_reference_from_disk(self) -> bool: - if not self._cached_layout_matches(): + if not self._cached_layout_matches() or not os.path.isfile(self._bank_index_path): return False - left = self._load_reference_image('ref_left.png', anchor.size(self.l_bbox)) - right = self._load_reference_image('ref_right.png', anchor.size(self.r_bbox)) - if left is None or right is None: - return False - self.ref_left = left - self.ref_right = right + with open(self._bank_index_path, 'r', encoding='utf-8') as handle: + counts = json.load(handle) + loaded = {} + for side, closed in self.banks: + name = _bank_name(side, closed) + shape = self._bank_shape(side, closed) + frames = [self._load_reference_image(f'{name}_{i:02d}.png', shape) + for i in range(counts.get(name, 0))] + if not frames or any(frame is None for frame in frames): + return False + loaded[(side, closed)] = frames + self.banks = loaded + self._distance_cache.clear() return True - def _load_reference_image(self, filename: str, expected_size: int): + def _bank_shape(self, side: str, door_closed: bool) -> tuple: + if door_closed: + left, top, right, bottom = self.windows[side] + return (anchor.size(bottom - top), anchor.size(right - left)) + size = anchor.size(self.l_bbox if side == 'left' else self.r_bbox) + return (size, size) + + def _load_reference_image(self, filename: str, expected_shape: tuple): path = os.path.join(self.reference_dir, filename) if not os.path.isfile(path): return None frame = cv2.imread(path, cv2.IMREAD_GRAYSCALE) - if frame is None or frame.shape != (expected_size, expected_size): + if frame is None or frame.shape != expected_shape: return None return frame.astype(np.float32) def save_reference_to_disk(self): - if self.ref_left is not None: - cv2.imwrite(os.path.join(self.reference_dir, 'ref_left.png'), self.ref_left.astype(np.uint8)) - if self.ref_right is not None: - cv2.imwrite(os.path.join(self.reference_dir, 'ref_right.png'), self.ref_right.astype(np.uint8)) + counts = {} + for (side, closed), frames in self.banks.items(): + name = _bank_name(side, closed) + counts[name] = len(frames) + for i, frame in enumerate(frames): + cv2.imwrite(os.path.join(self.reference_dir, f'{name}_{i:02d}.png'), frame.astype(np.uint8)) + with open(self._bank_index_path, 'w', encoding='utf-8') as handle: + json.dump(counts, handle) with open(self._layout_path, 'w', encoding='utf-8') as handle: handle.write(self._layout_fingerprint()) @@ -154,72 +215,35 @@ def clear_buffers(self): with self._buf_lock: self._left_buf.clear() self._right_buf.clear() - self._cam_buf.clear() + for buf in self._window_bufs.values(): + buf.clear() + self._map_buttons.clear() - def capture_camera_closed_reference(self): - frames = self._wait_for_frames(self._cam_buf) - self._ref_camera_closed = frames[0] if frames else None - - def _wait_for_frames(self, buf: collections.deque) -> list: - for _ in range(self._calib_frames): - with self._buf_lock: - frames = list(buf) - if frames: - return frames - time.sleep(self._calib_delay) - return [] - - def _capture_peak_reference(self, buf: collections.deque) -> np.ndarray: - best_frame = None - max_brightness = -1.0 - for _ in range(self._calib_frames): - with self._buf_lock: - frames = list(buf) - if not frames: - time.sleep(self._calib_delay) - continue - frame = frames[-1] - brightness = float(np.mean(frame)) - if brightness > max_brightness: - max_brightness = brightness - best_frame = frame - time.sleep(self._calib_delay) - return best_frame - - def camera_mse(self) -> float: - if self._ref_camera_closed is None: - return 0.0 - mse, current = self._get_min_mse_from_buffer(self._ref_camera_closed, self._cam_buf) - if current is None: - return 0.0 - return mse + def light_on(self, side: str) -> bool | None: + if not anchor.anchored(): + return None + motor = config.MOTOR_CALIBRATION + x, y = getattr(motor, f'{side}_light_button_x'), getattr(motor, f'{side}_light_button_y') + half = config.VISION_DYNAMICS.light_button_half_size + left, top = anchor.point(x - half, y - half) + right, bottom = anchor.point(x + half, y + half) + with mss.MSS() as screen: + patch = np.array(screen.grab({'left': left, 'top': top, + 'width': max(1, right - left), 'height': max(1, bottom - top)})) + gray = cv2.cvtColor(patch, cv2.COLOR_BGRA2GRAY) + return float(gray.mean()) > config.VISION_DYNAMICS.light_button_lit_level + + def map_buttons(self) -> list: + with self._buf_lock: + return list(self._map_buttons) def is_camera_up(self) -> bool: - if self._ref_camera_closed is None: - return False - mse, current = self._get_min_mse_from_buffer(self._ref_camera_closed, self._cam_buf) - if current is None: - return False - return mse > self._cam_mse_trigger + seen = self.map_buttons() + return len(seen) == self._map_buttons.maxlen and all(n >= self._min_buttons for n in seen) - def _mses_against(self, ref_frame: np.ndarray, buf: collections.deque): - with self._buf_lock: - frames = list(buf) - if not frames: - return [] - return [(float(np.mean((f - ref_frame) ** 2)), f) for f in frames] - - def _get_min_mse_from_buffer(self, ref_frame: np.ndarray, buf: collections.deque): - mses = self._mses_against(ref_frame, buf) - if not mses: - return float('inf'), None - return min(mses, key=lambda x: x[0]) - - def _get_peak_mse_from_buffer(self, ref_frame: np.ndarray, buf: collections.deque): - mses = self._mses_against(ref_frame, buf) - if not mses: - return 0.0, None - return max(mses, key=lambda x: x[0]) + def is_camera_down(self) -> bool: + seen = self.map_buttons() + return len(seen) == self._map_buttons.maxlen and all(n < self._min_buttons for n in seen) def _write_threat_evidence(self, side: str, current, reference, mse: float): stamp = f'{side}_{time.strftime("%H%M%S")}_{int(mse)}' @@ -242,34 +266,51 @@ def latest_patches(self) -> dict: 'right': self._right_buf[-1] if self._right_buf else None, } - def get_left_sensory_rate(self) -> float: - if self.ref_left is None: - return 0.0 - mse, current = self._get_peak_mse_from_buffer(self.ref_left, self._left_buf) - if current is None: + def get_left_sensory_rate(self, door_closed: bool = False) -> float: + return self._sensory_rate('left', door_closed) + + def get_right_sensory_rate(self, door_closed: bool = False) -> float: + return self._sensory_rate('right', door_closed) + + def evidence(self, side: str, door_closed: bool) -> tuple: + bank = self.banks[(side, door_closed)] + with self._buf_lock: + frames = list(self._buffer(side, door_closed)) + if not bank or not frames: + return None + nearest = [self._nearest(frame, bank, door_closed) for frame in frames] + middle = hallway_bank.median_index([distance for distance, _ in nearest]) + distance, reference = nearest[middle] + return distance, frames[middle], reference + + def _nearest(self, frame, bank: list, door_closed: bool) -> tuple: + key = (id(frame), door_closed) + cached = self._distance_cache.get(key) + if cached is None or cached[0] is not frame: + cached = (frame, hallway_bank.nearest(frame, bank)) + if len(self._distance_cache) > 64: + self._distance_cache.clear() + self._distance_cache[key] = cached + return cached[1] + + def _sensory_rate(self, side: str, door_closed: bool) -> float: + seen = self.evidence(side, door_closed) + if seen is None: return 0.0 - self._peak_mse['left'] = max(self._peak_mse['left'], mse) - self._last_mse['left'] = mse - if mse > self.mse_threshold: - if not self._threat_written_left: - self._write_threat_evidence('left', current, self.ref_left, mse) - self._threat_written_left = True + distance, current, reference = seen + self._peak_mse[side] = max(self._peak_mse[side], distance) + self._last_mse[side] = distance + written = f'_threat_written_{side}' + threshold = self.closed_mse_threshold if door_closed else self.mse_threshold + if distance > threshold: + if not getattr(self, written): + label = f'{side}_closed' if door_closed else side + self._write_threat_evidence(label, current, reference, distance) + setattr(self, written, True) return 1.0 - self._threat_written_left = False + setattr(self, written, False) return 0.0 - def get_right_sensory_rate(self) -> float: - if self.ref_right is None: - return 0.0 - mse, current = self._get_peak_mse_from_buffer(self.ref_right, self._right_buf) - if current is None: - return 0.0 - self._peak_mse['right'] = max(self._peak_mse['right'], mse) - self._last_mse['right'] = mse - if mse > self.mse_threshold: - if not self._threat_written_right: - self._write_threat_evidence('right', current, self.ref_right, mse) - self._threat_written_right = True - return 1.0 - self._threat_written_right = False - return 0.0 \ No newline at end of file + +def _bank_name(side: str, door_closed: bool) -> str: + return f'{side}_closed' if door_closed else f'{side}_open' diff --git a/src/flynaf/main.py b/src/flynaf/main.py index f1ad06d..f1180ff 100644 --- a/src/flynaf/main.py +++ b/src/flynaf/main.py @@ -37,7 +37,6 @@ async def _run(telemetry: ConnectomeTelemetry, trace: SessionRecorder, begin_nig view_ready = asyncio.Event() tracer = CascadeTracer.from_csr(engine.synapses) if config.BRAIN_VIEW.enabled else None feed = SpikeFeed(engine.num_neurons, tracer) - highlights = {'gaze': '--', 'gf_l': False, 'gf_r': False, 'camera': False} saccade = SaccadeRequest() tablet_feed = TabletFeed() tablet = TabletWatch(build_gaze(), tablet_feed, controller, telemetry, state) @@ -52,10 +51,10 @@ async def _run(telemetry: ConnectomeTelemetry, trace: SessionRecorder, begin_nig )) tg.create_task(engine_task( engine, vision, controller, state, shutdown, telemetry, calibration_done, - feed, highlights, saccade, trace, tablet, + feed, saccade, trace, tablet, )) tg.create_task(brain_view_task( - feed, highlights, shutdown, view_ready, vision.latest_patches, + feed, shutdown, view_ready, vision.latest_patches, )) diff --git a/src/flynaf/night/calibration.py b/src/flynaf/night/calibration.py index 230e93b..2b2c552 100644 --- a/src/flynaf/night/calibration.py +++ b/src/flynaf/night/calibration.py @@ -5,6 +5,8 @@ from flynaf.env.input_controller import FNAFController from flynaf.env.overlay import anchor_to_game, game_in_front, window_title from flynaf.env.vision import FNAFVision +from flynaf.night.lights import set_light +from flynaf.night.monitor import put_tablet_down from flynaf.night.motor import await_motor from flynaf.night.tablet.references import capture_camera_references @@ -19,19 +21,35 @@ async def calibrate_eye_references_live(vision: FNAFVision, controller: FNAFCont _log.info('T-%d', i) await asyncio.sleep(1.0) - await await_motor(controller.set_left_light(True)) - await asyncio.sleep(settle) - await asyncio.get_event_loop().run_in_executor(None, vision.capture_left_reference) - _log.info('left reference captured') - await await_motor(controller.set_left_light(False)) + for side in ('left', 'right'): + await calibrate_hallway(vision, controller, side, settle) + vision.save_reference_to_disk() - await await_motor(controller.set_right_light(True)) - await asyncio.sleep(settle) - await asyncio.get_event_loop().run_in_executor(None, vision.capture_right_reference) - _log.info('right reference captured') - await await_motor(controller.set_right_light(False)) - vision.save_reference_to_disk() +async def calibrate_hallway(vision: FNAFVision, controller: FNAFController, side: str, settle: float): + door_settle = settle + config.DOOR_DYNAMICS.motion_settle_sec + for door_closed in (False, True): + if door_closed: + await await_motor(getattr(controller, f'trigger_{side}_door')()) + await asyncio.sleep(door_settle) + await record_hallway_views(vision, controller, side, door_closed, settle) + await await_motor(getattr(controller, f'open_{side}_door')()) + await asyncio.sleep(door_settle) + + +async def record_hallway_views(vision: FNAFVision, controller: FNAFController, side: str, + door_closed: bool, settle: float): + if not await set_light(vision, controller, side, True): + _log.warning('the %s light did not come on while its references were recorded', side) + await asyncio.sleep(settle) + lit = await asyncio.to_thread(vision.capture_bank, side, door_closed, None, door_closed) + await set_light(vision, controller, side, False) + await asyncio.sleep(settle) + if door_closed: + _log.info('%s window behind the closed door: %d lit view', side, lit) + return + dark = await asyncio.to_thread(vision.capture_bank, side, door_closed, settle) + _log.info('%s hallway with the door open: %d lit views and %d dark', side, lit, dark) async def calibrate(vision: FNAFVision, controller: FNAFController, begin_night=None, tablet_feed=None): @@ -40,24 +58,27 @@ async def calibrate(vision: FNAFVision, controller: FNAFController, begin_night= if begin_night is None: await countdown_to_the_night(config.BRAIN_VIEW.start_countdown_sec) else: - await asyncio.get_event_loop().run_in_executor(None, begin_night) + await asyncio.to_thread(begin_night) confirm_game_in_front() + await start_with_the_tablet_down(vision, controller) if vision.load_reference_from_disk(): _log.info('loaded left/right eye references from disk, skipping live calibration') else: await calibrate_eye_references_live(vision, controller, settle) - await await_motor(controller.centre_view()) - await asyncio.sleep(settle) - vision.capture_camera_closed_reference() - _log.info('office reference captured with the view centred') - if tablet_feed is not None and await capture_camera_references(tablet_feed, controller, vision): _log.info('camera references captured for %s', ', '.join(camera.name for camera in config.TABLET_VISION.cameras)) +async def start_with_the_tablet_down(vision: FNAFVision, controller: FNAFController): + if await put_tablet_down(vision, controller): + return + _log.warning('the camera map is still on screen after %d gestures, lower the tablet by hand', + config.CAMERA_DETECTION.lower_gesture_attempts) + + async def countdown_to_the_night(seconds: float): _log.info('the panel is up. Go to the game and start the night, beginning in %.0fs', seconds) remaining = int(seconds) diff --git a/src/flynaf/night/doors.py b/src/flynaf/night/doors.py index f78150d..7fb3eb8 100644 --- a/src/flynaf/night/doors.py +++ b/src/flynaf/night/doors.py @@ -16,19 +16,22 @@ def __init__(self, controller, telemetry, state: SensoryState): self._state = state self._hold_leak = dynamics.hold_leak_per_frame self._release_threshold = dynamics.release_threshold - self._reopen_settle = dynamics.reopen_settle_sec + self._motion_settle = dynamics.motion_settle_sec self._nominal_fps = config.SIMULATION_PARAMS.target_fps self._motor_sec = config.FORAGING_PARAMS.motor_refractory_sec self._refrac = MotorRefrac() self.closed = {side: False for side in SIDES} self._closed_at = {side: 0.0 for side in SIDES} + self._renewed_at = {side: 0.0 for side in SIDES} + self._calm_at = {side: None for side in SIDES} self._hold = {side: 0.0 for side in SIDES} + self._moving = {side: None for side in SIDES} def refractory(self, side: str, now: float) -> bool: return now <= getattr(self._refrac, side) def on_giant_fiber(self, side: str, now: float) -> bool: - if self._state.camera_open: + if self._tablet_in_the_way(): return False moved = self.slam(side, now, 'giant fiber') is not None setattr(self._refrac, side, now + self._motor_sec) @@ -36,30 +39,62 @@ def on_giant_fiber(self, side: str, now: float) -> bool: def slam(self, side: str, now: float, cause: str): self._hold[side] = 1.0 + self._renewed_at[side] = now + self._calm_at[side] = None if self.closed[side]: + _log.info('%s %s fired behind the closed door, hold renewed', side, cause) + self._telemetry.record_door_hold_renewed(side, cause) return None _log.warning('%s %s fired, door closed', side, cause) self._telemetry.record_door_panic(side, cause) self.closed[side] = True + self._state.door_closed[side] = True self._closed_at[side] = now - return self._press(side, close=True) + return self._press(side, close=True, now=now) - def release_decayed(self, now: float, frame_elapsed: float): + def update(self, now: float, frame_elapsed: float): decay = self._hold_leak ** (frame_elapsed * self._nominal_fps) for side in SIDES: + self._shade_moving_door(side, now) if not self.closed[side]: continue - self._state.blind_until[side] = now + self._reopen_settle self._hold[side] *= decay - if self._hold[side] >= self._release_threshold or self._state.camera_open: + if self._hold[side] >= self._release_threshold: continue - held = now - self._closed_at[side] - _log.info('%s escape drive decayed after %.1fs, door released', side, held) - self._telemetry.record_door_release(side, held) - self._press(side, close=False) - self.closed[side] = False + if self._calm_at[side] is None: + self._calm_at[side] = now + if self._tablet_in_the_way() or not self._verified_clear(side): + continue + self._release(side, now) + + def _tablet_in_the_way(self) -> bool: + return self._state.camera_open or self._state.tablet_seen + + def _verified_clear(self, side: str) -> bool: + return self._state.cleared_at[side] > self._renewed_at[side] + + def _release(self, side: str, now: float): + held = now - self._closed_at[side] + waited = now - self._calm_at[side] + _log.info('%s escape drive decayed and the last look found the hallway empty, ' + 'door released after %.1fs', side, held) + self._telemetry.record_door_release(side, held, waited) + self.closed[side] = False + self._state.door_closed[side] = False + self._calm_at[side] = None + self._press(side, close=False, now=now) + + def _shade_moving_door(self, side: str, now: float): + moving = self._moving[side] + if moving is None: + return + self._state.blind_until[side] = now + self._motion_settle + if moving.is_set(): + self._moving[side] = None - def _press(self, side: str, close: bool): - if close: - return getattr(self._controller, f'trigger_{side}_door')() - return getattr(self._controller, f'open_{side}_door')() + def _press(self, side: str, close: bool, now: float): + action = f'trigger_{side}_door' if close else f'open_{side}_door' + done = getattr(self._controller, action)() + self._moving[side] = done + self._state.blind_until[side] = now + self._motion_settle + return done diff --git a/src/flynaf/night/lights.py b/src/flynaf/night/lights.py new file mode 100644 index 0000000..26b526a --- /dev/null +++ b/src/flynaf/night/lights.py @@ -0,0 +1,28 @@ +import asyncio +import logging + +from flynaf import config +from flynaf.night.motor import await_motor + +_log = logging.getLogger(__name__) + + +async def screen_disagrees(vision, side: str, on: bool) -> bool: + interval = config.VISION_DYNAMICS.light_check_interval_sec + for _ in range(2): + seen = await asyncio.to_thread(vision.light_on, side) + if seen is None or seen == on: + return False + await asyncio.sleep(interval) + return True + + +async def set_light(vision, controller, side: str, on: bool, corrections: int = 2) -> bool: + await await_motor(getattr(controller, f'set_{side}_light')(on)) + for _ in range(corrections): + if not await screen_disagrees(vision, side, on): + return True + _log.warning('the %s light button reads %s on screen, pressing it again', + side, 'off' if on else 'on') + await await_motor(getattr(controller, f'press_{side}_light')(on)) + return not await screen_disagrees(vision, side, on) diff --git a/src/flynaf/night/monitor.py b/src/flynaf/night/monitor.py index e5314c1..7d6fb9e 100644 --- a/src/flynaf/night/monitor.py +++ b/src/flynaf/night/monitor.py @@ -2,79 +2,46 @@ import logging from flynaf import clock, config -from flynaf.env.input_controller import FNAFController -from flynaf.env.vision import FNAFVision from flynaf.night.motor import await_motor from flynaf.night.state import SensoryState from flynaf.night.tablet.watch import IdleTablet _log = logging.getLogger(__name__) +ALREADY_THERE_SEC = 0.2 -async def office_comes_back( - vision: FNAFVision, controller, settle_sec: float, confirm_sec: float -) -> bool: - vision.clear_buffers() - deadline = clock.now() + confirm_sec - agreed = 0 - while clock.now() < deadline: - await asyncio.sleep(settle_sec / 2.0) - if controller is not None and controller.facing() != 'centre': - agreed = 0 - continue - if vision.is_camera_up(): - agreed = 0 - continue - agreed += 1 - if agreed >= 2: + +async def screen_shows(vision, up: bool, within_sec: float, poll_sec: float = 0.05) -> bool: + deadline = clock.now() + within_sec + while True: + if vision.is_camera_up() if up else vision.is_camera_down(): return True - return False + if clock.now() >= deadline: + return False + await asyncio.sleep(poll_sec) -async def reanchor_office_reference( - vision: FNAFVision, settle_sec: float, confirm_sec: float = 0.0, controller=None -) -> bool: - if not await office_comes_back(vision, controller, settle_sec, max(confirm_sec, settle_sec)): - return False - vision.capture_camera_closed_reference() - return True - - -async def lower_monitor( - vision: FNAFVision, controller: FNAFController, settle_sec: float, - confirm_sec: float, attempt: int, gesture_budget: int, -) -> bool: - gesture = None - if attempt < gesture_budget: - if attempt % 2 == 0: - gesture = 'sliding off the tablet bar' - await await_motor(controller.close_camera(force=True)) - else: - gesture = 'tapping the tablet bar' - await await_motor(controller.nudge_camera_bar()) - - if await reanchor_office_reference(vision, settle_sec, confirm_sec, controller): +async def put_tablet(vision, controller, up: bool, confirm_sec: float) -> bool: + if await screen_shows(vision, up, ALREADY_THERE_SEC): return True + await await_motor(controller.flip_tablet()) + return await screen_shows(vision, up, confirm_sec) - if gesture is None: - return False - _log.warning( - 'the monitor is still up after %s, attempt %d of %d, office patch %.0f mse from ' - 'its reference', gesture, attempt + 1, gesture_budget, vision.camera_mse(), - ) +async def put_tablet_down(vision, controller, attempts: int | None = None) -> bool: + detection = config.CAMERA_DETECTION + for _ in range(attempts or detection.lower_gesture_attempts): + if await put_tablet(vision, controller, False, detection.flip_confirm_sec): + return True return False -async def escape_from_tablet( - vision: FNAFVision, controller: FNAFController, lower_settle_sec: float, - settle_sec: float, confirm_sec: float, reflex, -) -> bool: - await await_motor(controller.close_camera(force=True)) - await asyncio.sleep(lower_settle_sec) - await reflex() - await await_motor(controller.centre_view()) - return await reanchor_office_reference(vision, settle_sec, confirm_sec, controller) +async def escape_from_tablet(vision, controller, confirm_sec: float, reflex) -> bool: + for _ in range(2): + if await put_tablet(vision, controller, False, confirm_sec): + await reflex() + return True + return False def release_reason(explore, bored: bool) -> str: @@ -86,71 +53,78 @@ def release_reason(explore, bored: bool) -> str: class MonitorControl: def __init__(self, vision, controller, telemetry, state: SensoryState, tablet=None): foraging = config.FORAGING_PARAMS + detection = config.CAMERA_DETECTION self._vision = vision self._controller = controller self._telemetry = telemetry self._state = state self._tablet = tablet or IdleTablet() - self._lower_settle_sec = config.TABLET_VISION.lower_settle_sec self._watch_max_sec = foraging.camera_watch_max_sec self._watch_min_sec = foraging.camera_watch_min_sec self._release_bias = foraging.camera_release_forage_bias self._refractory_sec = foraging.camera_refractory_sec - self._settle_sec = config.CAMERA_DETECTION.close_settle_sec - self._retry_sec = config.CAMERA_DETECTION.lower_retry_sec - self._confirm_sec = config.CAMERA_DETECTION.lower_confirm_sec - self._gesture_budget = config.CAMERA_DETECTION.lower_gesture_attempts + self._confirm_sec = detection.flip_confirm_sec + self._retry_sec = detection.lower_retry_sec + self._gesture_budget = detection.lower_gesture_attempts self._attempts = 0 self.is_open = False self.ready_at = 0.0 self._releasing = False - self._lowering = None + self._gesture = None self._retry_at = 0.0 self._opened_at = 0.0 self._cap_at = 0.0 + @property + def engaged(self) -> bool: + return self.is_open or self._gesture is not None + def update(self, now: float, explore, forage_bias: float, look_pending: bool): - self._collect_lowering(now) - if self._lowering is not None: + self._collect(now) + if self._gesture is not None: return if self.is_open: self._release(now, explore, forage_bias) return + if self._state.tablet_seen: + self._tidy(now) + return if explore.wants_monitor and now > self.ready_at and not look_pending: self._raise(now, explore) - def _raise(self, now: float, explore): - _log.info( - 'DNp09 %s, camera pull triggered', - 'fired' if explore.commanded - else f'drive reached the bound ({explore.drive:.2f})', - ) - self._telemetry.record_camera_pull(explore.commanded, explore.drive) - self._controller.open_camera() - self.is_open = True - self._state.camera_open = True - self._opened_at = now - self._cap_at = now + self._watch_max_sec - self._tablet.on_raise(now) - def escape(self, now: float, reflex) -> bool: - if not self.is_open or self._lowering is not None: + if not self.is_open or self._gesture is not None: return False if not self._releasing: self._telemetry.record_camera_release(now - self._opened_at, 'escape') self._releasing = True self._tablet.on_lower() - self._lowering = asyncio.create_task(escape_from_tablet( - self._vision, self._controller, self._lower_settle_sec, - self._settle_sec, self._confirm_sec, reflex, + self._start('lower', escape_from_tablet( + self._vision, self._controller, self._confirm_sec, reflex, )) - self._attempts = max(self._attempts, 1) return True + def _raise(self, now: float, explore): + _log.info( + 'DNp09 %s, camera pull triggered', + 'fired' if explore.commanded + else f'drive reached the bound ({explore.drive:.2f})', + ) + self._telemetry.record_camera_pull(explore.commanded, explore.drive) + self._start('raise', put_tablet(self._vision, self._controller, True, self._confirm_sec)) + + def _tidy(self, now: float): + if now < self._retry_at: + return + _log.warning('the tablet is on screen while the fly believes it is down, putting it away') + self._telemetry.record_tablet_found_up() + self._start('tidy', put_tablet(self._vision, self._controller, False, self._confirm_sec)) + def _release(self, now: float, explore, forage_bias: float): watched_for = now - self._opened_at if not self._releasing: - bored = watched_for >= self._watch_min_sec and ( + seen_for = now - max(self._opened_at, self._state.tablet_readable_at) + bored = seen_for >= self._watch_min_sec and ( explore.spent or forage_bias >= self._release_bias ) if now < self._cap_at and not bored: @@ -162,23 +136,36 @@ def _release(self, now: float, explore, forage_bias: float): return if self._attempts == self._gesture_budget: _log.warning( - 'neither gesture brought the monitor down in %d tries, so the fly has stopped ' - 'reaching for it and is just watching the screen. Lower the tablet by hand and ' - 'it picks up from there.', self._gesture_budget, - ) - self._lowering = asyncio.create_task( - lower_monitor( - self._vision, self._controller, self._settle_sec, self._confirm_sec, - self._attempts, self._gesture_budget, + 'the tablet has not come down after %d gestures, the fly keeps trying. ' + 'Lowering it by hand also works, the screen is what it believes.', + self._gesture_budget, ) - ) - self._attempts += 1 + self._start('lower', put_tablet(self._vision, self._controller, False, self._confirm_sec)) + + def _start(self, kind: str, gesture): + self._gesture = (kind, asyncio.create_task(gesture)) - def _collect_lowering(self, now: float): - if self._lowering is None or not self._lowering.done(): + def _collect(self, now: float): + if self._gesture is None or not self._gesture[1].done(): return - lowered = self._lowering.result() - self._lowering = None + kind, task = self._gesture + self._gesture = None + getattr(self, f'_after_{kind}')(now, task.result()) + + def _after_raise(self, now: float, raised: bool): + if not raised: + _log.warning('the tablet did not come up on screen, the camera pull is dropped') + self._telemetry.record_raise_missed() + self.ready_at = now + self._retry_sec + return + self.is_open = True + self._state.camera_open = True + self._opened_at = now + self._cap_at = now + self._watch_max_sec + self._tablet.on_raise(now) + + def _after_lower(self, now: float, lowered: bool): + self._attempts += 1 if not lowered: self._telemetry.record_monitor_stuck() self._retry_at = now + self._retry_sec @@ -190,3 +177,7 @@ def _collect_lowering(self, now: float): self._attempts = 0 self._state.camera_open = False self.ready_at = now + self._refractory_sec + + def _after_tidy(self, now: float, lowered: bool): + if not lowered: + self._retry_at = now + self._retry_sec diff --git a/src/flynaf/night/motor.py b/src/flynaf/night/motor.py index e195a44..b0eeb6e 100644 --- a/src/flynaf/night/motor.py +++ b/src/flynaf/night/motor.py @@ -5,7 +5,7 @@ async def await_motor(done, timeout_sec: float = 6.0): - finished = await asyncio.get_event_loop().run_in_executor(None, done.wait, timeout_sec) + finished = await asyncio.to_thread(done.wait, timeout_sec) if not finished: _log.warning('motor command did not complete within %.0fs', timeout_sec) return finished diff --git a/src/flynaf/night/panel.py b/src/flynaf/night/panel.py index 063da82..4737b87 100644 --- a/src/flynaf/night/panel.py +++ b/src/flynaf/night/panel.py @@ -155,21 +155,19 @@ def _launch_brain_view_window(port: int, page_height) -> bool: async def brain_view_task( - feed: SpikeFeed, highlights: dict, shutdown: asyncio.Event, view_ready: asyncio.Event, + feed: SpikeFeed, shutdown: asyncio.Event, view_ready: asyncio.Event, eyes=None, ): if not config.BRAIN_VIEW.enabled: view_ready.set() return - server = BrainViewServer(feed, highlights, eyes) + server = BrainViewServer(feed, eyes) try: port = await server.start() _log.info('brain view server listening on %d', port) if config.BRAIN_VIEW.launch_browser: - await asyncio.get_event_loop().run_in_executor( - None, _launch_brain_view_window, port, server.page_height - ) + await asyncio.to_thread(_launch_brain_view_window, port, server.page_height) view_ready.set() await shutdown.wait() except OSError as exc: diff --git a/src/flynaf/night/state.py b/src/flynaf/night/state.py index 754a1b0..1c86283 100644 --- a/src/flynaf/night/state.py +++ b/src/flynaf/night/state.py @@ -17,8 +17,11 @@ class SensoryState: l_sensory_count: int = 0 r_sensory_count: int = 0 forage_bias: float = 0.0 - office_centred: bool = True + tablet_seen: bool = False blind_until: dict = field(default_factory=lambda: {'left': 0.0, 'right': 0.0}) + door_closed: dict = field(default_factory=lambda: {'left': False, 'right': False}) + cleared_at: dict = field(default_factory=lambda: {'left': 0.0, 'right': 0.0}) + look_started: dict = field(default_factory=lambda: {'left': 0.0, 'right': 0.0}) tablet_camera: str | None = None tablet_readable_at: float = 0.0 tablet_drive: dict = field(default_factory=dict) diff --git a/src/flynaf/night/tablet/references.py b/src/flynaf/night/tablet/references.py index c4fa925..76a2067 100644 --- a/src/flynaf/night/tablet/references.py +++ b/src/flynaf/night/tablet/references.py @@ -2,7 +2,7 @@ import logging from flynaf import config -from flynaf.night.monitor import lower_monitor +from flynaf.night.monitor import put_tablet, put_tablet_down from flynaf.night.motor import await_motor _log = logging.getLogger(__name__) @@ -10,12 +10,10 @@ async def capture_camera_references(feed, controller, vision, params=None) -> bool: params = params or config.TABLET_VISION - await await_motor(controller.open_camera()) - await asyncio.sleep(params.raise_settle_sec) - if not vision.is_camera_up(): - _log.warning('the tablet did not come up for the camera references, so the fly ' - 'will raise it tonight without reading the feed') - await put_tablet_down(vision, controller) + if not await put_tablet(vision, controller, True, config.CAMERA_DETECTION.flip_confirm_sec): + _log.warning('the camera map did not appear for the camera references, so the fly ' + 'will raise the tablet tonight without reading the feed') + await _lower(vision, controller) return False feed.activate() @@ -23,23 +21,16 @@ async def capture_camera_references(feed, controller, vision, params=None) -> bo recorded = [await _record(feed, controller, camera.name, params) for camera in params.cameras] finally: feed.deactivate() - await put_tablet_down(vision, controller) + await _lower(vision, controller) return all(recorded) async def _record(feed, controller, camera: str, params) -> bool: await await_motor(controller.select_camera(camera)) await asyncio.sleep(params.switch_settle_sec) - return await asyncio.get_event_loop().run_in_executor(None, feed.capture_reference, camera) - - -async def put_tablet_down(vision, controller) -> bool: - detection = config.CAMERA_DETECTION - for attempt in range(detection.lower_gesture_attempts): - if await lower_monitor( - vision, controller, detection.close_settle_sec, detection.lower_confirm_sec, - attempt, detection.lower_gesture_attempts, - ): - return True - _log.warning('the tablet stayed up after the camera references, lower it by hand') - return False + return await asyncio.to_thread(feed.capture_reference, camera) + + +async def _lower(vision, controller): + if not await put_tablet_down(vision, controller): + _log.warning('the tablet stayed up after the camera references, lower it by hand') diff --git a/src/flynaf/night/tasks.py b/src/flynaf/night/tasks.py index e772fc6..e5c5c03 100644 --- a/src/flynaf/night/tasks.py +++ b/src/flynaf/night/tasks.py @@ -2,12 +2,13 @@ import logging from flynaf import clock, config -from flynaf.env.brain_view import SpikeFeed +from flynaf.env.brain_view import PanelFrame, SpikeFeed from flynaf.env.input_controller import FNAFController from flynaf.env.vision import FNAFVision from flynaf.night.calibration import calibrate from flynaf.night.doors import DoorControl from flynaf.night.engine import ConnectomeEngine +from flynaf.night.lights import set_light from flynaf.night.monitor import MonitorControl from flynaf.night.motor import await_motor from flynaf.night.state import SaccadeRequest, SensoryState @@ -27,14 +28,14 @@ async def vision_task( delay = config.VISION_DYNAMICS.capture_delay_sec senses = senses or BlindSenses() while not shutdown.is_set(): - state.office_centred = controller.facing() == 'centre' - cam_up = state.office_centred and vision.is_camera_up() + state.tablet_seen = vision.is_camera_up() now = clock.now() looking_left = state.check_left and now >= state.blind_until['left'] looking_right = state.check_right and now >= state.blind_until['right'] - state.left_rate = vision.get_left_sensory_rate() if looking_left else 0.0 - state.right_rate = vision.get_right_sensory_rate() if looking_right else 0.0 - state.cam_inhib = config.SIMULATION_PARAMS.base_sensory_rate_hz if cam_up else 0.0 + closed = state.door_closed + state.left_rate = vision.get_left_sensory_rate(closed['left']) if looking_left else 0.0 + state.right_rate = vision.get_right_sensory_rate(closed['right']) if looking_right else 0.0 + state.cam_inhib = config.SIMULATION_PARAMS.base_sensory_rate_hz if state.tablet_seen else 0.0 state.tablet_drive = senses.read(state, now) await asyncio.sleep(delay) @@ -44,6 +45,7 @@ async def observe_hallway( ) -> tuple: foraging = config.FORAGING_PARAMS vision.reset_peak_mse(side) + state.look_started[side] = clock.now() opened_at_frame = engine.frames opened_at_driven = engine.driven_frames[side] deadline = clock.now() + foraging.light_inspection_max_sec @@ -93,7 +95,7 @@ async def saccade_task( side = saccade.side saccade.busy = True - while state.camera_open and not shutdown.is_set(): + while (state.camera_open or state.tablet_seen) and not shutdown.is_set(): await asyncio.sleep(engine.frame_dt) if shutdown.is_set(): break @@ -101,19 +103,20 @@ async def saccade_task( _log.info('%s light check, %s (search drive %+.2f)', side, saccade.reason, saccade.drive) telemetry.record_light_saccade(side, saccade.drive, saccade.reason) - switch = controller.set_left_light if side == 'left' else controller.set_right_light - - await await_motor(switch(True)) + lit = await set_light(vision, controller, side, True) + if not lit: + _log.warning('the %s light would not come on, the hallway is read in the dark', side) await asyncio.sleep(config.FORAGING_PARAMS.light_activation_settle_sec) contrast, frames, driven, fired = await observe_hallway(engine, vision, state, side) - switch(False) + await set_light(vision, controller, side, False) threshold = config.FORAGING_PARAMS.mse_threshold _log.info('%s hallway read %.0f against a %.0f threshold, eye driven %d of %d frames%s', side, contrast, threshold, driven, frames, ', giant fiber answered' if fired else '') telemetry.record_look_contrast(side, contrast, frames, driven) + remember_a_clear_look(state, side, fired, driven, lit) saccade.ready_at = clock.now() + config.FORAGING_PARAMS.saccade_refractory_sec saccade.side = None @@ -129,7 +132,6 @@ async def engine_task( telemetry: ConnectomeTelemetry, calibration_done: asyncio.Event, feed: SpikeFeed, - highlights: dict, saccade: SaccadeRequest, trace: SessionRecorder | None = None, tablet=None, @@ -143,7 +145,6 @@ async def engine_task( explore = ExploreDrive() monitor = MonitorControl(vision, controller, telemetry, state, tablet) doors = DoorControl(controller, telemetry, state) - watchdog = InhibitionWatchdog(vision) last_frame_at = clock.now() while not shutdown.is_set(): @@ -152,8 +153,7 @@ async def engine_task( last_frame_at = now inputs = (state.left_rate > 0.0, state.right_rate > 0.0) - loop = asyncio.get_event_loop() - spikes = await loop.run_in_executor(None, engine.step, state, now) + spikes = await asyncio.to_thread(engine.step, state, now) read_spikes(engine, spikes, state) explore.update(engine.explore_membrane, state.explore_spike, frame_elapsed) @@ -163,15 +163,16 @@ async def engine_task( watch_the_tablet(tablet, monitor, doors, telemetry, state, now) request_saccade(search, saccade, monitor, engine, state, frame_elapsed, now, look_pending) - feed.publish(spikes[0].nonzero().flatten().cpu().numpy()) - fill_highlights(highlights, state, vision, doors, monitor, inputs) + feed.publish( + spikes[0].nonzero().flatten().cpu().numpy(), + panel_frame(state, vision, doors, monitor, inputs), + ) trace.frame( now - telemetry.start_time, state, engine, state.l_spike, state.r_spike, vision.peak_mse('left'), vision.peak_mse('right'), monitor.is_open, ) - watchdog.check(state, monitor, now) - doors.release_decayed(now, frame_elapsed) + doors.update(now, frame_elapsed) answer_giant_fibers(engine, doors, feed, state, now) remaining = engine.frame_dt - (clock.now() - now) @@ -189,6 +190,14 @@ async def warm_up(engine: ConnectomeEngine, telemetry: ConnectomeTelemetry): telemetry.record_warmup(frames, settled, engine.warmup_activity) +def remember_a_clear_look(state: SensoryState, side: str, fired: bool, driven: int, lit: bool = True): + if fired or driven > 0 or not lit: + return + if state.blind_until[side] > state.look_started[side]: + return + state.cleared_at[side] = clock.now() + + def read_spikes(engine, spikes, state: SensoryState): frame = spikes[0] state.l_spike = bool(frame[engine.l_motor_idx].any()) @@ -223,7 +232,8 @@ async def close_the_door(): def request_saccade(search: SearchDrive, saccade: SaccadeRequest, monitor: MonitorControl, engine, state: SensoryState, frame_elapsed: float, now: float, look_pending: bool): - can_look = not look_pending and not monitor.is_open and now >= saccade.ready_at + can_look = (not look_pending and not monitor.engaged and not state.tablet_seen + and now >= saccade.ready_at) wants = search.update( engine.eye_membrane_diff, state.l_sensory_count, state.r_sensory_count, frame_elapsed, now, can_look, @@ -252,50 +262,29 @@ def _strongest(drive: dict, prefix: str) -> float: return max((level for name, level in drive.items() if name.startswith(prefix)), default=0.0) -def fill_highlights(highlights: dict, state: SensoryState, vision: FNAFVision, - doors: DoorControl, monitor: MonitorControl, inputs: tuple): - highlights['gf_l'] = state.l_spike - highlights['gf_r'] = state.r_spike - highlights['camera'] = monitor.is_open - highlights['in_l'], highlights['in_r'] = inputs - highlights['look_l'] = state.check_left - highlights['look_r'] = state.check_right - highlights['mse_l'] = vision.last_mse('left') if state.check_left else None - highlights['mse_r'] = vision.last_mse('right') if state.check_right else None - highlights['mse_th'] = vision.mse_threshold - highlights['eye_l'] = state.l_sensory_count - highlights['eye_r'] = state.r_sensory_count - highlights['door_l'] = doors.closed['left'] - highlights['door_r'] = doors.closed['right'] - highlights['cam'] = state.tablet_camera - highlights['figure'] = _strongest(state.tablet_drive, 'figure') - highlights['loom'] = _strongest(state.tablet_drive, 'loom') - highlights['dnp09'] = state.explore_spike - highlights['dnp04_l'] = state.l_escape - highlights['dnp04_r'] = state.r_escape - highlights['gaze'] = 'LEFT' if state.check_left else 'RIGHT' if state.check_right else '--' - - -class InhibitionWatchdog: - def __init__(self, vision: FNAFVision): - self._vision = vision - self._warn_after = config.CAMERA_DETECTION.stuck_warn_sec - self._since = 0.0 - self._warned_at = 0.0 - - def check(self, state: SensoryState, monitor: MonitorControl, now: float): - if state.cam_inhib <= 0 or monitor.is_open: - self._since = 0.0 - return - if self._since == 0.0: - self._since = now - return - if now - self._since <= self._warn_after or now - self._warned_at <= 30.0: - return - _log.warning( - 'the office reference has disagreed with the screen for %.0fs at %.0f mse; ' - 'the inhibitors are being driven with the tablet down, which silences ' - 'the giant fiber and DNp09. Recapture it with the office on screen.', - now - self._since, self._vision.camera_mse(), - ) - self._warned_at = now +def panel_frame(state: SensoryState, vision: FNAFVision, doors: DoorControl, + monitor: MonitorControl, inputs: tuple) -> PanelFrame: + return PanelFrame( + gaze='LEFT' if state.check_left else 'RIGHT' if state.check_right else '--', + camera=monitor.is_open, + cam=state.tablet_camera, + gf_l=state.l_spike, + gf_r=state.r_spike, + in_l=inputs[0], + in_r=inputs[1], + look_l=state.check_left, + look_r=state.check_right, + mse_l=vision.last_mse('left') if state.check_left else None, + mse_r=vision.last_mse('right') if state.check_right else None, + mse_th=vision.mse_threshold, + eye_l=state.l_sensory_count, + eye_r=state.r_sensory_count, + door_l=doors.closed['left'], + door_r=doors.closed['right'], + figure=_strongest(state.tablet_drive, 'figure'), + loom=_strongest(state.tablet_drive, 'loom'), + dnp09=state.explore_spike, + dnp04_l=state.l_escape, + dnp04_r=state.r_escape, + ) + diff --git a/src/flynaf/scripts/debug_camera.py b/src/flynaf/scripts/debug_camera.py index aa6cae9..9a1410c 100644 --- a/src/flynaf/scripts/debug_camera.py +++ b/src/flynaf/scripts/debug_camera.py @@ -1,47 +1,25 @@ -import os import time -import cv2 -import mss -import numpy as np - from flynaf import config +from flynaf.env.overlay import anchor_to_game +from flynaf.env.vision import FNAFVision def main(): - cx = config.CAMERA_DETECTION.patch_x - cy = config.CAMERA_DETECTION.patch_y - csize = config.CAMERA_DETECTION.patch_size - offset = csize // 2 - bbox = {'top': cy - offset, 'left': cx - offset, 'width': csize, 'height': csize} - - print('Aguardando 3s para capturar ref-fechada. Volte para o jogo e fique na sala!') - time.sleep(3) - with mss.MSS() as sct: - ref_img = np.array(sct.grab(bbox)) - ref_gray = cv2.cvtColor(ref_img, cv2.COLOR_BGRA2GRAY).astype(np.float32) - - print('Referencia capturada! Gravando log em logs/mse_camera.txt...') - print('Abra a camera, espere 2 segundos, feche e depois de ALT+TAB para fechar o script.') - - os.makedirs('logs', exist_ok=True) - with open('logs/mse_camera.txt', 'w') as f: - f.write('--- LOG DE MSE DA CAMERA ---\n') - - with mss.MSS() as sct: - try: - while True: - cur_img = np.array(sct.grab(bbox)) - cur_gray = cv2.cvtColor(cur_img, cv2.COLOR_BGRA2GRAY).astype(np.float32) - mse = float(np.mean((cur_gray - ref_gray) ** 2)) - - with open('logs/mse_camera.txt', 'a') as f: - f.write(f'MSE: {mse:.1f}\n') - - print(f'\rGravando... (MSE Atual: {mse:.1f})', end='') - time.sleep(0.1) - except KeyboardInterrupt: - print('\nFinalizado pelo usuario.') + if not anchor_to_game(): + print('the game window was not found, the 1280x720 layout is read from the top left') + vision = FNAFVision() + needed = config.CAMERA_DETECTION.min_buttons + print(f'the tablet counts as up with {needed} or more camera buttons in view. Ctrl+C to stop.') + try: + while True: + seen = vision.map_buttons() + verdict = 'UP' if vision.is_camera_up() else 'down' if vision.is_camera_down() else 'changing' + print(f'\rbuttons {seen} tablet {verdict:<8}', end='', flush=True) + time.sleep(0.1) + except KeyboardInterrupt: + print('\nstopped') + if __name__ == '__main__': main() diff --git a/src/flynaf/scripts/debug_reflex.py b/src/flynaf/scripts/debug_reflex.py index 96c6685..5594252 100644 --- a/src/flynaf/scripts/debug_reflex.py +++ b/src/flynaf/scripts/debug_reflex.py @@ -26,17 +26,13 @@ def main(): return print(f'threat threshold (MSE): {threshold:.0f}') - print(f'camera-up threshold : {config.CAMERA_DETECTION.mse_trigger:.0f}') - print('Capturing the camera-closed reference in 3s, stay in the office.') - time.sleep(3.0) - vision.capture_camera_closed_reference() print('Ready. Hold a light on and watch the side that is lit. Ctrl+C to stop.\n') print(f"{'left MSE':>10} {'right MSE':>10} {'tablet':>8} verdict") try: while True: - left, _ = vision._get_peak_mse_from_buffer(vision.ref_left, vision._left_buf) - right, _ = vision._get_peak_mse_from_buffer(vision.ref_right, vision._right_buf) + left = (vision.evidence('left', door_closed=False) or (0.0,))[0] + right = (vision.evidence('right', door_closed=False) or (0.0,))[0] up = vision.is_camera_up() verdict = [] diff --git a/src/flynaf/scripts/measure_door_window.py b/src/flynaf/scripts/measure_door_window.py new file mode 100644 index 0000000..1a8f547 --- /dev/null +++ b/src/flynaf/scripts/measure_door_window.py @@ -0,0 +1,72 @@ +import argparse +import time + +from flynaf import config +from flynaf.env.input_controller import FNAFController, start_worker +from flynaf.env.overlay import anchor_to_game +from flynaf.env.vision import FNAFVision + + +def sample_window(vision: FNAFVision, side: str, seconds: float) -> list: + readings = [] + deadline = time.perf_counter() + seconds + while time.perf_counter() < deadline: + seen = vision.evidence(side, door_closed=True) + if seen is not None: + readings.append(seen[0]) + time.sleep(0.1) + return readings + + +def measure(side: str, seconds: float) -> list: + controller = FNAFController() + settle = config.FORAGING_PARAMS.light_activation_settle_sec + config.DOOR_DYNAMICS.motion_settle_sec + light = getattr(controller, f'set_{side}_light') + vision = FNAFVision() + if not vision.load_reference_from_disk(): + return [] + + getattr(controller, f'trigger_{side}_door')().wait(6.0) + time.sleep(settle) + light(True).wait(6.0) + time.sleep(settle) + readings = sample_window(vision, side, seconds) + light(False).wait(6.0) + getattr(controller, f'open_{side}_door')().wait(6.0) + return readings + + +def summary(readings: list) -> str: + ordered = sorted(readings) + return (f'{len(ordered)} readings, min {ordered[0]:.0f}, ' + f'median {ordered[len(ordered) // 2]:.0f}, peak {ordered[-1]:.0f}') + + +def main(): + parser = argparse.ArgumentParser( + description='Close one door, light its window and report how far the window strays from ' + 'the nearest closed-door view calibrated at midnight. Run it once with the hallway empty and once with ' + 'Bonnie or Chica outside, then set closed_door_mse_threshold between the two.', + ) + parser.add_argument('side', choices=('left', 'right')) + parser.add_argument('--seconds', type=float, default=5.0) + args = parser.parse_args() + + anchor_to_game() + start_worker() + print(f'measuring the {args.side} window in 3s, leave the game in front') + time.sleep(3.0) + readings = measure(args.side, args.seconds) + if not readings: + print('no closed-door references in logs/vision_reference, run one night first so the ' + 'live calibration captures them') + return + + threshold = config.FORAGING_PARAMS.closed_door_mse_threshold + print(f'{args.side} window behind the closed door: {summary(readings)}') + print(f'closed_door_mse_threshold is {threshold:.0f}, ' + f'{sum(1 for mse in readings if mse > threshold)} readings above it') + + +if __name__ == '__main__': + main() diff --git a/src/flynaf/telemetry.py b/src/flynaf/telemetry.py index 9f6c5c2..e37c446 100644 --- a/src/flynaf/telemetry.py +++ b/src/flynaf/telemetry.py @@ -14,11 +14,14 @@ def __init__(self): 'right_door_panics': 0, 'left_door_releases': 0, 'right_door_releases': 0, + 'left_hold_renewals': 0, + 'right_hold_renewals': 0, 'camera_pulls': 0, 'left_light_saccades': 0, 'right_light_saccades': 0 } self.door_holds = [] + self.verification_waits = [] self.saccade_reasons = {'drift': 0, 'evidence': 0, 'guard': 0} self.camera_causes = {'spike': 0, 'drive': 0} self.camera_watches = [] @@ -33,6 +36,8 @@ def __init__(self): self.inhibited_frames = 0 self.looking_frames = 0 self.monitor_stuck = 0 + self.tablet_found_up = 0 + self.raises_missed = 0 self.saccade_gaps = [] self._last_saccade_at = None self._first_frame_at = None @@ -59,11 +64,22 @@ def record_tablet_escape(self, side: str, camera: str): f"[{t:>6.1f}s] DNp04 {side} fired on a looming shape in CAM {camera}. Tablet dropped for the door." ) - def record_door_release(self, side: str, held_sec: float): + def record_door_hold_renewed(self, side: str, cause: str): + t = clock.now() - self.start_time + self.stats[f'{side}_hold_renewals'] += 1 + self.events.append( + f"[{t:>6.1f}s] {cause.capitalize()} fired behind the closed {side} door, the threat is still there. Hold renewed." + ) + + def record_door_release(self, side: str, held_sec: float, waited_sec: float = 0.0): t = clock.now() - self.start_time self.stats[f'{side}_door_releases'] += 1 self.door_holds.append(held_sec) - self.events.append(f"[{t:>6.1f}s] Escape drive decayed after {held_sec:.1f}s. {side.capitalize()} door reopened.") + self.verification_waits.append(waited_sec) + self.events.append( + f"[{t:>6.1f}s] Escape drive decayed and a look found the {side} hallway empty. " + f"{side.capitalize()} door reopened after {held_sec:.1f}s." + ) def record_camera_pull(self, commanded: bool, drive: float): t = clock.now() - self.start_time @@ -111,7 +127,21 @@ def record_monitor_stuck(self): t = clock.now() - self.start_time self.monitor_stuck += 1 self.events.append( - f"[{t:>6.1f}s] Monitor stayed up after both lowering gestures. Retrying." + f"[{t:>6.1f}s] The camera map was still on screen after the lowering gesture. Retrying." + ) + + def record_tablet_found_up(self): + t = clock.now() - self.start_time + self.tablet_found_up += 1 + self.events.append( + f"[{t:>6.1f}s] The camera map was on screen while the fly believed the tablet was down. Putting it away." + ) + + def record_raise_missed(self): + t = clock.now() - self.start_time + self.raises_missed += 1 + self.events.append( + f"[{t:>6.1f}s] The camera map did not appear after the raising gesture. Camera pull dropped." ) CAUSES = { @@ -166,6 +196,10 @@ def dump_report(self): mean_hold = sum(self.door_holds) / len(self.door_holds) f.write(f"Mean Door Hold : {mean_hold:.1f}s\n") f.write(f"Total Door-Closed Time : {sum(self.door_holds):.1f}s\n") + waits = sorted(self.verification_waits) + f.write(f"Wait For The Clearing Look : median {waits[len(waits) // 2]:.1f}s, max {waits[-1]:.1f}s\n") + f.write(f"Holds Renewed By A Look : {self.stats['left_hold_renewals']} left, " + f"{self.stats['right_hold_renewals']} right\n") f.write(f"Spontaneous Camera Pulls : {self.stats['camera_pulls']}\n") if self.camera_watches: watches = len(self.camera_watches) @@ -174,7 +208,9 @@ def dump_report(self): f.write(f"Monitor Raised By : {self.camera_causes['spike']} DNp09 spikes, {self.camera_causes['drive']} accumulated drive\n") by = self.camera_release_reasons f.write(f"Monitor Lowered By : {by['drive']} drive faded, {by['search']} hallway won, {by['cap']} power cap, {by['escape']} looming escape\n") - f.write(f"Left Light Checks : {self.stats['left_light_saccades']}\n") + f.write(f"Tablet Gestures Missed : {self.raises_missed} raises, {self.monitor_stuck} lowerings retried\n") + f.write(f"Tablet Found Up Unasked : {self.tablet_found_up}\n") + f.write(f"Left Light Checks : {self.stats['left_light_saccades']}\n") f.write(f"Right Light Checks : {self.stats['right_light_saccades']}\n\n") f.write("--- Who Decided To Look ---\n") diff --git a/src/flynaf/tuning.py b/src/flynaf/tuning.py index 444f8d4..27542d1 100644 --- a/src/flynaf/tuning.py +++ b/src/flynaf/tuning.py @@ -38,9 +38,10 @@ class Knob: ), Knob( 'mse_threshold', 'FORAGING_PARAMS', 'mse_threshold', - 1500.0, 200.0, 20000.0, False, - 'How different a lit hallway must look before the eye drives its cluster. ' - 'Lower means more door slams and more false alarms.', + 40.0, 10.0, 100.0, False, + 'How far a hallway may be from the nearest view calibrated at midnight, lit, ' + 'flickering or dark, before the eye drives its cluster. Bonnie in the doorway ' + 'measured 118, so a value near that leaves the fly blind to him.', ), Knob( 'starvation_sec', 'SEARCH_DYNAMICS', 'starvation_sec', diff --git a/tests/test_camera_reference_refresh.py b/tests/test_camera_reference_refresh.py deleted file mode 100644 index 5d1f231..0000000 --- a/tests/test_camera_reference_refresh.py +++ /dev/null @@ -1,74 +0,0 @@ -import asyncio -from unittest.mock import MagicMock - -from flynaf.night.monitor import reanchor_office_reference - - -def _office_visible(): - vision = MagicMock() - vision.is_camera_up.return_value = False - return vision - - -def _centred(): - controller = MagicMock() - controller.facing.return_value = 'centre' - return controller - - -def test_reanchor_captures_reference_only_after_the_screen_agrees(): - vision = _office_visible() - - async def scenario(): - task = asyncio.create_task(reanchor_office_reference(vision, 0.02, 0.3, _centred())) - vision.capture_camera_closed_reference.assert_not_called() - await task - - asyncio.run(scenario()) - - vision.capture_camera_closed_reference.assert_called_once() - - -def test_reanchor_needs_more_than_one_clean_reading(): - vision = _office_visible() - - asyncio.run(reanchor_office_reference(vision, 0.02, 0.3, _centred())) - - assert vision.is_camera_up.call_count >= 2 - - -def test_reanchor_clears_buffers_before_capturing_reference(): - call_order = [] - vision = _office_visible() - vision.clear_buffers.side_effect = lambda: call_order.append('clear') - vision.capture_camera_closed_reference.side_effect = lambda: call_order.append('capture') - - asyncio.run(reanchor_office_reference(vision, 0.02, 0.3, _centred())) - - assert call_order == ['clear', 'capture'] - - -def test_reanchor_refuses_to_capture_the_tablet_as_the_office(): - vision = MagicMock() - vision.is_camera_up.return_value = True - - reanchored = asyncio.run(reanchor_office_reference(vision, 0.02, 0.3, _centred())) - - assert reanchored is False - vision.capture_camera_closed_reference.assert_not_called() - - -def test_reanchor_does_not_touch_camera_open_state(): - vision = _office_visible() - state = MagicMock(camera_open='untouched-sentinel') - - asyncio.run(reanchor_office_reference(vision, 0.02, 0.3, _centred())) - - assert state.camera_open == 'untouched-sentinel' - - -if __name__ == '__main__': - for name, fn in sorted(globals().items()): - if name.startswith('test_'): - fn() - print('ok') diff --git a/tests/test_closed_door_window.py b/tests/test_closed_door_window.py new file mode 100644 index 0000000..d97f165 --- /dev/null +++ b/tests/test_closed_door_window.py @@ -0,0 +1,92 @@ +import collections +import threading +from unittest.mock import patch + +import numpy as np +import pytest + +from flynaf import config +from flynaf.env import vision as vision_module +from flynaf.env.vision import FNAFVision + +HEIGHT, WIDTH = 36, 20 +SHADOW_MEASURED = (420, 263, 517, 541) + + +def window_scene(streak: bool, head: bool) -> np.ndarray: + frame = np.full((HEIGHT, WIDTH), 40.0, dtype=np.float32) + if streak: + frame[:, 11:13] = 200.0 + if head: + frame[6:20, 9:17] = 5.0 + return frame + + +LIT_EMPTY = window_scene(streak=True, head=False) +FLICKER = window_scene(streak=False, head=False) +BONNIE = window_scene(streak=False, head=True) + + +class Replay: + def __init__(self, frames): + self._frames = list(frames) + self._shown = 0 + + def __bool__(self): + return True + + def __getitem__(self, _index): + frame = self._frames[min(self._shown, len(self._frames) - 1)] + self._shown += 1 + return frame + + +def eye() -> FNAFVision: + vision = FNAFVision.__new__(FNAFVision) + vision._buf_lock = threading.Lock() + vision.mse_threshold = config.FORAGING_PARAMS.mse_threshold + vision.closed_mse_threshold = config.FORAGING_PARAMS.closed_door_mse_threshold + vision._peak_mse = {'left': 0.0, 'right': 0.0} + vision._last_mse = {'left': 0.0, 'right': 0.0} + vision._threat_written_left = True + vision._threat_written_right = True + vision._distance_cache = {} + vision._sweep_sec = 0.05 + vision._bank_tolerance = config.VISION_DYNAMICS.bank_tolerance_mse + vision._capture_delay = 0.0 + vision.banks = {(side, closed): [] for side in ('left', 'right') for closed in (False, True)} + vision._window_bufs = {side: collections.deque(maxlen=5) for side in ('left', 'right')} + return vision + + +def record_the_closed_window(vision: FNAFVision, frames: list): + vision._window_bufs['left'] = Replay(frames) + with patch.object(vision_module.time, 'sleep', lambda _s: None): + vision.capture_bank('left', True, None, True) + vision._window_bufs['left'] = collections.deque(maxlen=5) + + +def test_the_closed_window_bank_keeps_only_the_lit_view(): + vision = eye() + + record_the_closed_window(vision, [FLICKER, LIT_EMPTY, FLICKER, FLICKER]) + + assert len(vision.banks[('left', True)]) == 1 + assert np.array_equal(vision.banks[('left', True)][0], LIT_EMPTY) + + +@pytest.mark.parametrize('seen, driven', [(LIT_EMPTY, 0.0), (BONNIE, 1.0)]) +def test_bonnie_behind_the_closed_door_drives_the_eye_and_the_empty_window_does_not(seen, driven): + vision = eye() + record_the_closed_window(vision, [FLICKER, LIT_EMPTY]) + vision._window_bufs['left'].extend([seen] * 3) + + assert vision.get_left_sensory_rate(door_closed=True) == driven + + +def test_the_left_window_holds_the_whole_shadow_measured_on_night_2(): + left, top, right, bottom = config.VISION_CALIBRATION.left_window + shadow_left, shadow_top, shadow_right, shadow_bottom = SHADOW_MEASURED + + assert left <= shadow_left and right >= shadow_right + assert top <= shadow_top and bottom >= shadow_bottom diff --git a/tests/test_corollary_discharge.py b/tests/test_corollary_discharge.py index 65c4af6..61a8513 100644 --- a/tests/test_corollary_discharge.py +++ b/tests/test_corollary_discharge.py @@ -1,5 +1,7 @@ import asyncio +import pytest + from flynaf import config from flynaf.night import tasks from flynaf.night.state import SensoryState @@ -7,36 +9,26 @@ FULL_RATE = config.SIMULATION_PARAMS.base_sensory_rate_hz -class PannedVision: +class MapOnScreen: def __init__(self, camera_up): self._camera_up = camera_up def is_camera_up(self): return self._camera_up - def get_left_sensory_rate(self): + def get_left_sensory_rate(self, door_closed=False): return 0.0 - def get_right_sensory_rate(self): + def get_right_sensory_rate(self, door_closed=False): return 0.0 -class FacingController: - def __init__(self, side): - self.side = side - - def facing(self): - return self.side - - -def _one_pass(camera_up, facing): - vision = PannedVision(camera_up) - controller = FacingController(facing) - state = SensoryState() +def one_pass(camera_up, state=None): + state = state or SensoryState() shutdown = asyncio.Event() async def scenario(): - task = asyncio.create_task(tasks.vision_task(vision, controller, state, shutdown)) + task = asyncio.create_task(tasks.vision_task(MapOnScreen(camera_up), None, state, shutdown)) await asyncio.sleep(0.1) shutdown.set() await task @@ -45,32 +37,15 @@ async def scenario(): return state -def test_a_pan_the_fly_caused_itself_is_not_read_as_a_raised_tablet(): - """The camera-up detector compares a fixed patch of the office against a - reference taken with the office centred. Holding a hallway light scrolls the - office toward that side, which moves the whole patch and reads exactly like - a tablet unless the fly discounts its own movement.""" - for side in ('left', 'right', 'panning'): - state = _one_pass(camera_up=True, facing=side) - assert state.cam_inhib == 0.0, f'the inhibitors were driven while facing {side}' - assert state.office_centred is False - +@pytest.mark.parametrize('camera_up, inhibitors', [(True, FULL_RATE), (False, 0.0)]) +def test_the_inhibitors_follow_the_camera_map_on_screen(camera_up, inhibitors): + state = one_pass(camera_up) -def test_a_raised_tablet_still_drives_the_inhibitors_when_centred(): - state = _one_pass(camera_up=True, facing='centre') + assert state.tablet_seen is camera_up + assert state.cam_inhib == inhibitors - assert state.cam_inhib == FULL_RATE - assert state.office_centred is True - - -def test_a_centred_office_with_the_tablet_down_leaves_the_brain_free(): - state = _one_pass(camera_up=False, facing='centre') - assert state.cam_inhib == 0.0 +def test_the_screen_wins_over_what_the_fly_believes(): + state = one_pass(camera_up=True, state=SensoryState(camera_open=False)) - -if __name__ == '__main__': - for name, fn in sorted(globals().items()): - if name.startswith('test_'): - fn() - print('ok') + assert state.cam_inhib == FULL_RATE diff --git a/tests/test_door_release.py b/tests/test_door_release.py index ed74d4f..9be4e11 100644 --- a/tests/test_door_release.py +++ b/tests/test_door_release.py @@ -24,12 +24,10 @@ def test_hold_outlives_the_motor_refractory(): assert seconds > config.FORAGING_PARAMS.motor_refractory_sec -def test_reopen_settle_is_shorter_than_the_hold(): - """The blind window after reopening has to end well before the next hold - could start, otherwise the fly reopens into a threat it cannot see.""" +def test_the_blind_window_of_a_moving_door_is_shorter_than_the_hold(): seconds = decay_frames_to_release() / config.SIMULATION_PARAMS.target_fps - assert config.DOOR_DYNAMICS.reopen_settle_sec < seconds / 2 + assert config.DOOR_DYNAMICS.motion_settle_sec < seconds / 2 def test_a_renewed_spike_recharges_the_hold_to_full(): @@ -45,6 +43,6 @@ def test_a_renewed_spike_recharges_the_hold_to_full(): if __name__ == '__main__': test_hold_lasts_a_few_seconds_at_the_engine_frame_rate() test_hold_outlives_the_motor_refractory() - test_reopen_settle_is_shorter_than_the_hold() + test_the_blind_window_of_a_moving_door_is_shorter_than_the_hold() test_a_renewed_spike_recharges_the_hold_to_full() print('ok') diff --git a/tests/test_door_verification.py b/tests/test_door_verification.py new file mode 100644 index 0000000..fbc7ac2 --- /dev/null +++ b/tests/test_door_verification.py @@ -0,0 +1,228 @@ +import asyncio +import collections +import threading +from unittest.mock import patch + +import numpy as np +import pytest + +from flynaf import config +from flynaf.env.vision import FNAFVision +from flynaf.night import calibration +from flynaf.night.doors import DoorControl +from flynaf.night.state import SensoryState +from flynaf.night.tasks import remember_a_clear_look +from flynaf.telemetry import ConnectomeTelemetry + +FRAME = 0.1 +SIZE = 16 + + +class MotorEvent: + def __init__(self, finished=True): + self.finished = finished + + def wait(self, timeout=None): + return self.finished + + def is_set(self): + return self.finished + + +class DoorButtons: + def __init__(self, finished=True): + self.calls = [] + self.finished = finished + + def __getattr__(self, name): + if name.startswith('_') or 'door' not in name: + raise AttributeError(name) + + def press(): + self.calls.append(name) + return MotorEvent(self.finished) + return press + + +def closed_door(side='left', finished=True): + state = SensoryState() + buttons = DoorButtons(finished) + telemetry = ConnectomeTelemetry() + doors = DoorControl(buttons, telemetry, state) + doors.on_giant_fiber(side, now=1.0) + return doors, buttons, telemetry, state + + +def run_frames(doors, start: float, seconds: float): + now = start + for _ in range(int(seconds / FRAME)): + now += FRAME + doors.update(now, FRAME) + return now + + +def test_a_decayed_hold_without_a_clearing_look_keeps_the_door_shut(): + doors, buttons, _, state = closed_door() + + run_frames(doors, 1.0, 20.0) + + assert doors.closed['left'] + assert state.door_closed['left'] + assert buttons.calls == ['trigger_left_door'] + + +def test_a_clear_look_after_the_slam_opens_the_door_once_the_hold_has_decayed(): + doors, buttons, telemetry, state = closed_door() + state.cleared_at['left'] = 2.0 + + now = run_frames(doors, 1.0, 1.0) + assert doors.closed['left'], 'the door opened while the defensive state was still high' + + run_frames(doors, now, 10.0) + assert not doors.closed['left'] + assert not state.door_closed['left'] + assert buttons.calls == ['trigger_left_door', 'open_left_door'] + assert telemetry.stats['left_door_releases'] == 1 + + +def test_a_clear_look_from_before_the_slam_does_not_count(): + doors, buttons, _, state = closed_door() + state.cleared_at['left'] = 0.5 + + run_frames(doors, 1.0, 20.0) + + assert doors.closed['left'] + + +def test_a_threat_seen_through_the_window_renews_the_hold_and_voids_older_clear_looks(): + doors, buttons, telemetry, state = closed_door() + state.cleared_at['left'] = 2.0 + now = run_frames(doors, 1.0, 2.0) + + assert doors.on_giant_fiber('left', now) is False + run_frames(doors, now, 20.0) + + assert doors.closed['left'] + assert telemetry.stats['left_hold_renewals'] == 1 + assert buttons.calls == ['trigger_left_door'] + + +def test_the_door_waits_for_the_tablet_to_come_down(): + doors, buttons, _, state = closed_door() + state.cleared_at['left'] = 2.0 + state.camera_open = True + + now = run_frames(doors, 1.0, 20.0) + assert doors.closed['left'] + + state.camera_open = False + run_frames(doors, now, FRAME) + assert not doors.closed['left'] + + +@pytest.mark.parametrize('finished, blind', [(False, True), (True, False)]) +def test_the_eye_is_blind_only_while_the_door_is_moving(finished, blind): + doors, _, _, state = closed_door(finished=finished) + + now = run_frames(doors, 1.0, 2.0) + + assert (state.blind_until['left'] > now) is blind + + +@pytest.mark.parametrize('fired, driven, blind_until, lit, clear', [ + (False, 0, 0.0, True, True), + (True, 0, 0.0, True, False), + (False, 3, 0.0, True, False), + (False, 0, 99.0, True, False), + (False, 0, 0.0, False, False), +]) +def test_only_a_sighted_lit_look_that_found_nothing_counts_as_clear(fired, driven, blind_until, lit, clear): + state = SensoryState() + state.look_started['right'] = 10.0 + state.blind_until['right'] = blind_until + + remember_a_clear_look(state, 'right', fired, driven, lit) + + assert (state.cleared_at['right'] > 0.0) is clear + + +def eye_with_both_references(): + vision = FNAFVision.__new__(FNAFVision) + vision._buf_lock = threading.Lock() + vision.mse_threshold = config.FORAGING_PARAMS.mse_threshold + vision.closed_mse_threshold = config.FORAGING_PARAMS.closed_door_mse_threshold + vision._peak_mse = {'left': 0.0, 'right': 0.0} + vision._last_mse = {'left': 0.0, 'right': 0.0} + vision._threat_written_left = True + vision._threat_written_right = True + vision._distance_cache = {} + open_view = np.zeros((SIZE, SIZE), dtype=np.float32) + shut_view = np.full((SIZE, SIZE), 120.0, dtype=np.float32) + vision.banks = { + (side, closed): [shut_view if closed else open_view] + for side in ('left', 'right') for closed in (False, True) + } + vision._left_buf = collections.deque([shut_view] * 3, maxlen=5) + vision._right_buf = collections.deque([shut_view] * 3, maxlen=5) + vision._window_bufs = { + side: collections.deque([shut_view] * 3, maxlen=5) for side in ('left', 'right') + } + return vision + + +@pytest.mark.parametrize('side', ['left', 'right']) +def test_the_eye_compares_a_closed_door_with_what_a_closed_door_looks_like(side): + vision = eye_with_both_references() + read = getattr(vision, f'get_{side}_sensory_rate') + + assert read(door_closed=False) == 1.0, 'the open-door reference should see the door itself' + assert read(door_closed=True) == 0.0, 'an empty window behind a closed door read as a threat' + + +@pytest.mark.parametrize('side', ['left', 'right']) +def test_a_shadow_in_the_window_behind_a_closed_door_drives_the_eye(side): + vision = eye_with_both_references() + shadow = np.full((SIZE, SIZE), 120.0 - 60.0, dtype=np.float32) + vision._window_bufs[side].extend([shadow] * 3) + + assert getattr(vision, f'get_{side}_sensory_rate')(door_closed=True) == 1.0 + + +class CalibrationRig: + def __init__(self): + self.order = [] + + def __getattr__(self, name): + if name.startswith('_'): + raise AttributeError(name) + + if name == 'light_on': + return lambda side: None + + def act(*args): + self.order.append(name if not args or name == 'capture_bank' else f'{name}({args[0]})') + if name == 'capture_bank': + self.order[-1] = f'bank({args[0]}, closed={args[1]}, {"dark" if len(args) == 3 else "lit"})' + return 1 + return MotorEvent() + return act + + +@pytest.mark.parametrize('side', ['left', 'right']) +def test_the_open_hallway_is_recorded_lit_and_dark_and_the_closed_window_only_lit(side): + rig = CalibrationRig() + + async def no_wait(_seconds): + return None + + with patch.object(calibration.asyncio, 'sleep', new=no_wait): + asyncio.run(calibration.calibrate_hallway(rig, rig, side, settle=0.0)) + + assert rig.order == [ + f'set_{side}_light(True)', f'bank({side}, closed=False, lit)', + f'set_{side}_light(False)', f'bank({side}, closed=False, dark)', + f'trigger_{side}_door', + f'set_{side}_light(True)', f'bank({side}, closed=True, lit)', + f'set_{side}_light(False)', + f'open_{side}_door', + ] diff --git a/tests/test_hallway_detection.py b/tests/test_hallway_detection.py index f7f5c1d..f1e7290 100644 --- a/tests/test_hallway_detection.py +++ b/tests/test_hallway_detection.py @@ -2,86 +2,80 @@ import threading import numpy as np +import pytest from flynaf import config +from flynaf.env import hallway_bank from flynaf.env.vision import FNAFVision SIZE = 32 +LIT = np.full((SIZE, SIZE), 20.0, dtype=np.float32) +FLICKER = np.full((SIZE, SIZE), 8.0, dtype=np.float32) +DARK = np.full((SIZE, SIZE), 5.0, dtype=np.float32) -def _eye(frames): +def dim_figure_in_the_doorway() -> np.ndarray: + frame = LIT.copy() + frame[4:28, 6:20] = 42.0 + return frame + + +def eye(frames, bank=(LIT, FLICKER, DARK)) -> FNAFVision: vision = FNAFVision.__new__(FNAFVision) vision._buf_lock = threading.Lock() vision.mse_threshold = config.FORAGING_PARAMS.mse_threshold + vision.closed_mse_threshold = config.FORAGING_PARAMS.closed_door_mse_threshold vision._peak_mse = {'left': 0.0, 'right': 0.0} vision._last_mse = {'left': 0.0, 'right': 0.0} vision._threat_written_left = True vision._threat_written_right = True - vision.ref_left = np.zeros((SIZE, SIZE), dtype=np.float32) - vision.ref_right = np.zeros((SIZE, SIZE), dtype=np.float32) + vision._distance_cache = {} + vision.banks = {(side, closed): list(bank) for side in ('left', 'right') for closed in (False, True)} vision._left_buf = collections.deque(frames, maxlen=5) vision._right_buf = collections.deque(frames, maxlen=5) return vision -def _intruder(): - return np.full((SIZE, SIZE), 80.0, dtype=np.float32) - - -def _dim_flicker(): - return np.zeros((SIZE, SIZE), dtype=np.float32) - - -def test_one_dim_flicker_frame_used_to_hide_an_intruder(): - """The hallway light flickers, so any buffer holds a frame that resembles - the unlit reference. Reading the minimum meant the least alarming frame - decided, and a lit-up Bonnie reported clear on most samples.""" - buffer = [_intruder(), _intruder(), _dim_flicker(), _intruder(), _intruder()] - vision = _eye(buffer) - - quiet, _ = vision._get_min_mse_from_buffer(vision.ref_left, vision._left_buf) - loudest, _ = vision._get_peak_mse_from_buffer(vision.ref_left, vision._left_buf) +def test_a_dim_figure_too_faint_for_the_old_threshold_drives_the_eye(): + figure = dim_figure_in_the_doorway() + assert hallway_bank.frame_distance(figure, LIT) < 1500.0 - assert quiet < vision.mse_threshold - assert loudest > vision.mse_threshold + assert eye([figure] * 5).get_left_sensory_rate() == 1.0 -def test_the_eye_now_reports_the_strongest_recent_evidence(): - buffer = [_intruder(), _intruder(), _dim_flicker(), _intruder(), _intruder()] - vision = _eye(buffer) - - assert vision.get_left_sensory_rate() == 1.0 - assert vision.get_right_sensory_rate() == 1.0 - - -def test_an_empty_hallway_still_reads_clear(): - vision = _eye([_dim_flicker()] * 5) +@pytest.mark.parametrize('frames', [ + [LIT] * 5, + [FLICKER] * 5, + [DARK] * 5, + [LIT, FLICKER, LIT, DARK, LIT], +]) +def test_every_view_calibrated_at_midnight_reads_as_empty(frames): + vision = eye(frames) assert vision.get_left_sensory_rate() == 0.0 assert vision.peak_mse('left') < vision.mse_threshold -def test_the_peak_is_recorded_even_when_below_the_threshold(): - faint = np.full((SIZE, SIZE), 5.0, dtype=np.float32) - vision = _eye([faint] * 5) +def test_a_flicker_frame_cannot_hide_a_figure_that_stays(): + figure = dim_figure_in_the_doorway() - vision.get_left_sensory_rate() + assert eye([figure, figure, FLICKER, figure, figure]).get_right_sensory_rate() == 1.0 - assert vision.peak_mse('left') == 25.0 +def test_a_single_odd_frame_does_not_make_a_threat(): + figure = dim_figure_in_the_doorway() -def test_the_tablet_trigger_sits_between_the_measured_office_and_tablet(): - """Measured on 2026-09-17: the office patch reads 182 to 294 with the tablet - down and 2703 to 3646 with it up. The old trigger of 100 sat inside the - office's own variation, so a lowering could never be confirmed.""" - trigger = config.CAMERA_DETECTION.mse_trigger + assert eye([LIT, LIT, figure, LIT, LIT]).get_left_sensory_rate() == 0.0 - assert trigger > 294 - assert trigger < 2703 + +def test_a_flicker_left_out_of_the_bank_is_still_absorbed_when_it_is_brief(): + vision = eye([LIT, FLICKER, LIT, FLICKER, LIT], bank=(LIT, DARK)) + + assert vision.get_left_sensory_rate() == 0.0 -if __name__ == '__main__': - for name, fn in sorted(globals().items()): - if name.startswith('test_'): - fn() - print('ok') +def test_the_threshold_sits_under_bonnie_as_measured_in_a_live_night(): + """On 2026-10-01 Bonnie in the lit left doorway was 124 from the lit view and + 118 from the flicker view, while every empty frame matched one of them exactly.""" + assert 0.0 < config.FORAGING_PARAMS.mse_threshold < 118.0 / 2 + assert 0.0 < config.FORAGING_PARAMS.closed_door_mse_threshold < 118.0 diff --git a/tests/test_light_check.py b/tests/test_light_check.py new file mode 100644 index 0000000..9a7fff2 --- /dev/null +++ b/tests/test_light_check.py @@ -0,0 +1,89 @@ +import asyncio + +import pytest + +from flynaf.night import lights + + +class Done: + def wait(self, timeout=None): + return True + + +class Lamp: + def __init__(self, lit=False, stuck=False, blink_reads=0): + self.lit = lit + self.stuck = stuck + self.blink_reads = blink_reads + self.believed = False + self.presses = 0 + + def set_left_light(self, on): + if self.believed != on: + self._toggle() + self.believed = on + return Done() + + def press_left_light(self, state): + self.presses += 1 + self._toggle() + self.believed = state + return Done() + + def _toggle(self): + if not self.stuck: + self.lit = not self.lit + + def light_on(self, side): + if self.blink_reads: + self.blink_reads -= 1 + return not self.lit + return self.lit + + +@pytest.fixture(autouse=True) +def quick_reads(monkeypatch): + async def no_wait(_seconds): + return None + monkeypatch.setattr(lights.asyncio, 'sleep', no_wait) + + +def switch(lamp, on): + return asyncio.run(lights.set_light(lamp, lamp, 'left', on)) + + +@pytest.mark.parametrize('on', [True, False]) +def test_a_light_in_step_with_the_controller_needs_no_correction(on): + lamp = Lamp(lit=False) + if not on: + switch(lamp, True) + + assert switch(lamp, on) is True + assert lamp.lit is on + assert lamp.presses == 0 + + +def test_a_light_left_on_before_the_night_is_put_right_by_the_screen(): + lamp = Lamp(lit=True) + + assert switch(lamp, True) is True + assert lamp.lit is True + assert lamp.presses == 1 + + assert switch(lamp, False) is True + assert lamp.lit is False + assert lamp.presses == 1 + + +def test_one_odd_reading_does_not_press_the_button(): + lamp = Lamp(lit=False, blink_reads=1) + + assert switch(lamp, True) is True + assert lamp.presses == 0 + + +def test_a_dead_button_is_reported_after_the_corrections_run_out(): + lamp = Lamp(lit=False, stuck=True) + + assert switch(lamp, True) is False + assert lamp.presses == 2 diff --git a/tests/test_monitor_lowering.py b/tests/test_monitor_lowering.py index 64c6ff8..18d0229 100644 --- a/tests/test_monitor_lowering.py +++ b/tests/test_monitor_lowering.py @@ -1,59 +1,41 @@ import asyncio -from flynaf import config +import pytest + from flynaf.night import monitor from flynaf.night.state import SensoryState from flynaf.telemetry import ConnectomeTelemetry -class _ImmediateEvent: +class Done: def wait(self, timeout=None): return True -CONFIRM = config.CAMERA_DETECTION.lower_confirm_sec -BUDGET = config.CAMERA_DETECTION.lower_gesture_attempts - - -class ScriptedVision: - def __init__(self, still_up_for: int): - self.still_up_for = still_up_for - self.checks = 0 - self.captures = 0 - self.clears = 0 - - def clear_buffers(self): - self.clears += 1 +class Screen: + def __init__(self, up=False, ignored_flips=0): + self.up = up + self.ignored_flips = ignored_flips def is_camera_up(self): - self.checks += 1 - return self.checks <= self.still_up_for - - def capture_camera_closed_reference(self): - self.captures += 1 + return self.up - def camera_mse(self): - return 4200.0 + def is_camera_down(self): + return not self.up -class RecordingController: - def __init__(self): - self.calls = [] +class Bar: + def __init__(self, screen: Screen): + self.screen = screen + self.flips = 0 - def facing(self): - return 'centre' - - def close_camera(self, force=False): - self.calls.append('close_camera') - return _ImmediateEvent() - - def nudge_camera_bar(self): - self.calls.append('nudge_camera_bar') - return _ImmediateEvent() - - def open_camera(self): - self.calls.append('open_camera') - return _ImmediateEvent() + def flip_tablet(self): + self.flips += 1 + if self.screen.ignored_flips: + self.screen.ignored_flips -= 1 + else: + self.screen.up = not self.screen.up + return Done() class StubExplore: @@ -64,160 +46,168 @@ def __init__(self, wants=False, spent=True): self.drive = 1.2 -def _lower(vision, controller, attempt=0, confirm=0.05): - return asyncio.run(monitor.lower_monitor( - vision, controller, settle_sec=0.02, confirm_sec=confirm, - attempt=attempt, gesture_budget=BUDGET, - )) - +@pytest.fixture(autouse=True) +def quick_confirmation(monkeypatch): + monkeypatch.setattr(monitor, 'ALREADY_THERE_SEC', 0.0) -def test_the_reference_is_not_recaptured_while_the_tablet_is_still_up(): - vision = ScriptedVision(still_up_for=99) - reanchored = asyncio.run(monitor.reanchor_office_reference(vision, 0.02, 0.05)) +def put(screen, up): + bar = Bar(screen) + reached = asyncio.run(monitor.put_tablet(screen, bar, up, confirm_sec=0.05)) + return reached, bar.flips - assert reanchored is False - assert vision.captures == 0 +@pytest.mark.parametrize('start_up, wanted_up', [(False, True), (True, False)]) +def test_one_gesture_puts_the_tablet_where_it_is_wanted(start_up, wanted_up): + screen = Screen(up=start_up) -def test_the_reference_is_recaptured_once_the_office_is_back(): - vision = ScriptedVision(still_up_for=0) + assert put(screen, wanted_up) == (True, 1) + assert screen.up is wanted_up - reanchored = asyncio.run(monitor.reanchor_office_reference(vision, 0.02, 0.05)) - assert reanchored is True - assert vision.captures == 1 - assert vision.clears == 1 +@pytest.mark.parametrize('up', [True, False]) +def test_no_gesture_when_the_screen_already_shows_the_wanted_state(up): + assert put(Screen(up=up), up) == (True, 0) -def test_one_early_reading_does_not_condemn_a_close_that_worked(): - """The gestures alternate, so declaring failure too soon makes the next - attempt tap the bar and put the tablet straight back up. That flap is what - left the fly blind for fifty seconds on 2026-09-17.""" - vision = ScriptedVision(still_up_for=1) - controller = RecordingController() +def test_a_gesture_the_game_ignored_is_reported_as_a_miss(): + screen = Screen(up=True, ignored_flips=1) - lowered = _lower(vision, controller) + assert put(screen, False) == (False, 1) + assert screen.up is True - assert lowered is True - assert controller.calls == ['close_camera'] - assert vision.captures == 1 +def test_putting_the_tablet_down_retries_until_the_screen_agrees(): + screen = Screen(up=True, ignored_flips=2) + bar = Bar(screen) -def test_each_attempt_makes_exactly_one_gesture(): - vision = ScriptedVision(still_up_for=99) - controller = RecordingController() + lowered = asyncio.run(monitor.put_tablet_down(screen, bar, attempts=4)) - assert _lower(vision, controller, attempt=0) is False - assert controller.calls == ['close_camera'] + assert lowered is True + assert bar.flips == 3 - assert _lower(vision, RecordingController(), attempt=1) is False - second = RecordingController() - _lower(vision, second, attempt=1) - assert second.calls == ['nudge_camera_bar'] +def control(screen, state=None): + state = state or SensoryState() + telemetry = ConnectomeTelemetry() + bar = Bar(screen) + control_ = monitor.MonitorControl(screen, bar, telemetry, state) + control_._confirm_sec = 0.05 + return control_, state, telemetry, bar -def test_the_fly_stops_reaching_once_the_budget_is_spent(): - vision = ScriptedVision(still_up_for=99) - controller = RecordingController() +async def settle(control_, now, explore): + for _ in range(50): + control_._state.tablet_seen = control_._vision.up + control_.update(now, explore, 0.0, look_pending=False) + if control_._gesture is None: + return + await asyncio.sleep(0.01) - lowered = _lower(vision, controller, attempt=BUDGET) - assert lowered is False - assert controller.calls == [] +def test_the_tablet_does_not_rise_while_a_look_is_pending(): + control_, state, _, bar = control(Screen()) + async def scenario(): + control_.update(1000.0, StubExplore(wants=True), 0.0, look_pending=True) -def test_a_tablet_lowered_by_hand_is_picked_up_without_a_gesture(): - vision = ScriptedVision(still_up_for=0) - controller = RecordingController() + asyncio.run(scenario()) - lowered = _lower(vision, controller, attempt=BUDGET) + assert bar.flips == 0 + assert control_.is_open is False - assert lowered is True - assert controller.calls == [] - assert vision.captures == 1 +def test_the_fly_only_believes_the_tablet_is_up_once_the_map_is_on_screen(): + control_, state, telemetry, bar = control(Screen()) -async def _settle_lowering(monitor): - while monitor._lowering is not None: - await asyncio.wait_for(asyncio.shield(monitor._lowering), timeout=10.0) - await asyncio.sleep(0) - monitor.update(2000.0, StubExplore(wants=False, spent=True), 0.0, look_pending=False) + async def scenario(): + await settle(control_, 1000.0, StubExplore(wants=True)) + asyncio.run(scenario()) -def _monitor(vision, controller): - state = SensoryState() - telemetry = ConnectomeTelemetry() - return monitor.MonitorControl(vision, controller, telemetry, state), state, telemetry + assert control_.is_open is True + assert state.camera_open is True + assert telemetry.stats['camera_pulls'] == 1 -def test_the_tablet_does_not_rise_while_a_look_is_pending(): - monitor, state, _ = _monitor(ScriptedVision(0), RecordingController()) +def test_a_raise_the_game_ignored_leaves_the_fly_believing_the_tablet_is_down(): + control_, state, telemetry, _ = control(Screen(ignored_flips=1)) async def scenario(): - monitor.update(1000.0, StubExplore(wants=True), 0.0, look_pending=True) + await settle(control_, 1000.0, StubExplore(wants=True)) asyncio.run(scenario()) - assert monitor.is_open is False + assert control_.is_open is False assert state.camera_open is False + assert telemetry.raises_missed == 1 -def test_the_tablet_rises_when_nothing_else_is_running(): - controller = RecordingController() - monitor, state, telemetry = _monitor(ScriptedVision(0), controller) +def test_a_lowering_counts_only_when_the_map_has_left_the_screen(): + screen = Screen() + control_, state, telemetry, _ = control(screen) async def scenario(): - monitor.update(1000.0, StubExplore(wants=True), 0.0, look_pending=False) + await settle(control_, 1000.0, StubExplore(wants=True)) + screen.ignored_flips = 1 + await settle(control_, 1002.0, StubExplore(wants=False, spent=True)) + assert control_.is_open is True + await settle(control_, 1010.0, StubExplore(wants=False, spent=True)) asyncio.run(scenario()) - assert monitor.is_open is True - assert state.camera_open is True - assert 'open_camera' in controller.calls - assert telemetry.stats['camera_pulls'] == 1 + assert telemetry.monitor_stuck == 1 + assert control_.is_open is False + assert state.camera_open is False + assert screen.up is False -def test_the_tablet_stays_up_until_the_screen_confirms_it_came_down(): - vision = ScriptedVision(still_up_for=99) - controller = RecordingController() - monitor, state, telemetry = _monitor(vision, controller) +def test_a_tablet_found_on_screen_behind_the_flys_back_is_put_away(): + screen = Screen(up=True) + control_, _, telemetry, bar = control(screen) async def scenario(): - monitor.update(1000.0, StubExplore(wants=True), 0.0, look_pending=False) - bored = StubExplore(wants=False, spent=True) - monitor.update(1002.0, bored, 0.0, look_pending=False) - await _settle_lowering(monitor) + await settle(control_, 1000.0, StubExplore(wants=False)) asyncio.run(scenario()) - assert monitor.is_open is True - assert state.camera_open is True - assert telemetry.monitor_stuck == 1 + assert screen.up is False + assert bar.flips == 1 + assert telemetry.tablet_found_up == 1 + assert control_.is_open is False + +@pytest.mark.parametrize('ignored, door_closed', [(0, True), (2, False)]) +def test_the_escape_closes_the_door_only_after_the_tablet_is_down(ignored, door_closed): + screen = Screen() + control_, _, _, _ = control(screen) + closed = [] -def test_a_confirmed_lowering_clears_the_open_state(): - vision = ScriptedVision(still_up_for=0) - controller = RecordingController() - monitor, state, telemetry = _monitor(vision, controller) + async def reflex(): + assert screen.up is False + closed.append(True) async def scenario(): - monitor.update(1000.0, StubExplore(wants=True), 0.0, look_pending=False) - bored = StubExplore(wants=False, spent=True) - monitor.update(1002.0, bored, 0.0, look_pending=False) - await _settle_lowering(monitor) + await settle(control_, 1000.0, StubExplore(wants=True)) + screen.ignored_flips = ignored + control_.escape(1001.0, reflex) + await control_._gesture[1] asyncio.run(scenario()) - assert monitor.is_open is False - assert state.camera_open is False - assert telemetry.monitor_stuck == 0 - assert len(telemetry.camera_watches) == 1 + assert bool(closed) is door_closed + +@pytest.mark.parametrize('watched, still_open', [(1.0, True), (1.9, True), (2.1, False)]) +def test_the_fly_only_tires_of_a_camera_after_it_could_read_it(watched, still_open): + screen = Screen() + control_, state, _, _ = control(screen) + + async def scenario(): + await settle(control_, 1000.0, StubExplore(wants=True)) + state.tablet_readable_at = 1000.0 + 1.2 + await settle(control_, 1000.0 + watched, StubExplore(wants=False, spent=True)) + + asyncio.run(scenario()) -if __name__ == '__main__': - for name, fn in sorted(globals().items()): - if name.startswith('test_'): - fn() - print('ok') + assert control_.is_open is still_open diff --git a/tests/test_motor_pan_settle.py b/tests/test_motor_pan_settle.py index 53281cc..532450e 100644 --- a/tests/test_motor_pan_settle.py +++ b/tests/test_motor_pan_settle.py @@ -24,9 +24,6 @@ def test_pan_side_classifies_every_real_button(): def test_the_office_reports_which_way_it_faces(): - """The camera-up detector reads a fixed patch of the office, so a pan the - fly caused itself looks exactly like a raised tablet unless it knows it is - mid-turn.""" mock_desktop = MagicMock() with patch.object(input_controller, 'desktop', mock_desktop), patch.object(input_controller.time, 'sleep'): @@ -43,7 +40,7 @@ def test_the_office_reports_which_way_it_faces(): cmd_queue.join() assert controller.facing() == 'right' - controller.open_camera() + controller.flip_tablet() cmd_queue.join() assert controller.facing() == 'centre' diff --git a/tests/test_overlay_placement.py b/tests/test_overlay_placement.py index 3662fe5..cb5adbf 100644 --- a/tests/test_overlay_placement.py +++ b/tests/test_overlay_placement.py @@ -1,5 +1,7 @@ from unittest.mock import patch +import pytest + from flynaf import config from flynaf.env import anchor from flynaf.env.overlay import ( @@ -16,6 +18,13 @@ EDGE_MIN_WIDTH = 265 +@pytest.fixture(autouse=True) +def unanchored(): + anchor.set_game_rect(None) + yield + anchor.set_game_rect(None) + + def _beside(game_rect, screen=(1920, 1080)): with patch.object(anchor, 'game_rect', return_value=game_rect): return beside_game_rect(screen[0], screen[1], 420, 620, 8) diff --git a/tests/test_search_loop_wiring.py b/tests/test_search_loop_wiring.py index 976fc5f..bef7a89 100644 --- a/tests/test_search_loop_wiring.py +++ b/tests/test_search_loop_wiring.py @@ -71,6 +71,7 @@ def __init__(self, state=None): self.calls = [] self.log = [] self.tablet = [] + self.tablet_up = False self._state = state def _done(self, name): @@ -84,7 +85,7 @@ def _done(self, name): elif self._state.check_right: gaze = 'right' self.log.append((name, gaze)) - self.tablet.append((name, bool(self._state and self._state.camera_open))) + self.tablet.append((name, self.tablet_up or bool(self._state and self._state.camera_open))) return _ImmediateEvent() def set_left_light(self, on): @@ -105,28 +106,25 @@ def trigger_left_door(self): def trigger_right_door(self): return self._done('trigger_right_door') - def centre_view(self): - return self._done('centre_view') - - def open_camera(self): - return self._done('open_camera') - - def close_camera(self, force=False): - return self._done('close_camera') - - def nudge_camera_bar(self): - return self._done('nudge_camera_bar') + def flip_tablet(self): + done = self._done('flip_tablet') + self.tablet_up = not self.tablet_up + return done class _ImmediateEvent: def wait(self, timeout=None): return True + def is_set(self): + return True + class FakeVision: - mse_threshold = 1500.0 + mse_threshold = 40.0 - def __init__(self): + def __init__(self, controller=None): + self._controller = controller self.peaks = {'left': 0.0, 'right': 0.0} self.ref_left = None self.ref_right = None @@ -141,15 +139,18 @@ def last_mse(self, side): return self.peaks[side] def is_camera_up(self): - return False + return bool(self._controller and self._controller.tablet_up) - def get_left_sensory_rate(self): - return 0.0 + def is_camera_down(self): + return not self.is_camera_up() + + def light_on(self, side): + return None - def get_right_sensory_rate(self): + def get_left_sensory_rate(self, door_closed=False): return 0.0 - def camera_mse(self): + def get_right_sensory_rate(self, door_closed=False): return 0.0 def clear_buffers(self): @@ -158,12 +159,9 @@ def clear_buffers(self): def load_reference_from_disk(self): return True - def capture_camera_closed_reference(self): - pass - class FakeFeed: - def publish(self, indices): + def publish(self, indices, panel=None): pass def trace_escape(self, side, target, sources, door_moved): @@ -208,16 +206,16 @@ async def stop(): shutdown.set() vision_task = asyncio.create_task(tasks.vision_task( - FakeVision(), controller, state, shutdown, + FakeVision(controller), controller, state, shutdown, )) engine_task = asyncio.create_task(tasks.engine_task( - engine, FakeVision(), controller, state, shutdown, telemetry, - calibrated, FakeFeed(), {}, saccade, + engine, FakeVision(controller), controller, state, shutdown, telemetry, + calibrated, FakeFeed(), saccade, )) view_ready = asyncio.Event() view_ready.set() saccade_task = asyncio.create_task(tasks.saccade_task( - engine, FakeVision(), controller, state, shutdown, telemetry, calibrated, + engine, FakeVision(controller), controller, state, shutdown, telemetry, calibrated, saccade, view_ready, )) watcher = asyncio.create_task(watch_gaze()) @@ -237,7 +235,7 @@ async def stop(): def _run(seconds, engine=None): - with patch.object(calibration, 'countdown_to_the_night', new=_office_is_up): + with patch.object(calibration, 'countdown_to_the_night', new=_office_is_up), patch.object(calibration, 'game_in_front', return_value=0): return asyncio.run(_play(seconds, engine)) diff --git a/tests/test_startup_readiness.py b/tests/test_startup_readiness.py index c7be617..86ba2bb 100644 --- a/tests/test_startup_readiness.py +++ b/tests/test_startup_readiness.py @@ -3,7 +3,7 @@ from flynaf import config from flynaf.env import overlay -from flynaf.night import calibration +from flynaf.night import calibration, monitor GAME = 789498 EDITOR = 66778 @@ -112,7 +112,6 @@ def test_a_launcher_hook_replaces_the_countdown(): vision = MagicMock() vision.load_reference_from_disk.return_value = True controller = MagicMock() - controller.centre_view.return_value.wait.return_value = True with patch.object(calibration, 'countdown_to_the_night', new=_refuse), \ patch.object(calibration, 'game_in_front', return_value=0): @@ -127,27 +126,27 @@ async def _refuse(_seconds): raise AssertionError('the countdown ran although the launcher starts the night') -def test_the_office_reference_is_taken_with_the_view_centred(): - order = [] +def test_the_night_starts_by_putting_away_a_tablet_already_on_screen(): + screen = {'up': True} vision = MagicMock() vision.load_reference_from_disk.return_value = True - vision.capture_camera_closed_reference.side_effect = lambda: order.append('capture') + vision.is_camera_up.side_effect = lambda: screen['up'] + vision.is_camera_down.side_effect = lambda: not screen['up'] - controller = MagicMock() - - def centre(): - order.append('centre') + def flip(): + screen['up'] = not screen['up'] event = MagicMock() event.wait.return_value = True return event - controller.centre_view.side_effect = centre + controller = MagicMock() + controller.flip_tablet.side_effect = flip - with patch.object(calibration, 'countdown_to_the_night', new=_ready), \ - patch.object(calibration, 'game_in_front', return_value=0): + with patch.object(calibration, 'countdown_to_the_night', new=_ready), patch.object(calibration, 'game_in_front', return_value=0), patch.object(monitor, 'ALREADY_THERE_SEC', 0.0): asyncio.run(calibration.calibrate(vision, controller)) - assert order == ['centre', 'capture'] + assert controller.flip_tablet.call_count == 1 + assert screen['up'] is False async def _ready(_seconds): diff --git a/tests/test_tablet_escape.py b/tests/test_tablet_escape.py index a219c0c..1e54026 100644 --- a/tests/test_tablet_escape.py +++ b/tests/test_tablet_escape.py @@ -1,5 +1,7 @@ import asyncio +import pytest + from flynaf.night import monitor, tasks from flynaf.night.doors import DoorControl from flynaf.night.state import SensoryState @@ -10,36 +12,41 @@ class _Done: def wait(self, timeout=None): return True + def is_set(self): + return True + class RecordingController: def __init__(self): self.calls = [] - - def facing(self): - return 'centre' + self.tablet_up = False def __getattr__(self, name): - if name.startswith('_'): + if name.startswith('_') or name == 'tablet_up': raise AttributeError(name) def record(*args, **kwargs): self.calls.append(name) + if name == 'flip_tablet': + self.tablet_up = not self.tablet_up return _Done() return record -class OfficeVision: - def clear_buffers(self): - pass +class ScreenVision: + def __init__(self, controller: RecordingController): + self._controller = controller def is_camera_up(self): - return False + return self._controller.tablet_up + + def is_camera_down(self): + return not self._controller.tablet_up - def capture_camera_closed_reference(self): - pass - def camera_mse(self): - return 0.0 +@pytest.fixture(autouse=True) +def quick_confirmation(monkeypatch): + monkeypatch.setattr(monitor, 'ALREADY_THERE_SEC', 0.0) class StubTablet: @@ -66,19 +73,24 @@ class StubExplore: drive = 1.2 -def _raised(escape_side='left'): +async def _raised(escape_side='left'): state = SensoryState() controller = RecordingController() telemetry = ConnectomeTelemetry() tablet = StubTablet(escape_side) - control = monitor.MonitorControl(OfficeVision(), controller, telemetry, state, tablet) - control.update(1.0, StubExplore(), 0.0, look_pending=False) + control = monitor.MonitorControl(ScreenVision(controller), controller, telemetry, state, tablet) + control._confirm_sec = 0.05 + for _ in range(100): + control.update(1.0, StubExplore(), 0.0, look_pending=False) + if control.is_open: + break + await asyncio.sleep(0.01) doors = DoorControl(controller, telemetry, state) return control, doors, controller, telemetry, state, tablet async def _escape_and_settle(escape_side='left'): - control, doors, controller, telemetry, state, tablet = _raised(escape_side) + control, doors, controller, telemetry, state, tablet = await _raised(escape_side) tasks.watch_the_tablet(tablet, control, doors, telemetry, state, now=2.0) for _ in range(300): control.update(3.0, StubExplore(), 0.0, look_pending=False) @@ -92,10 +104,10 @@ def _run(escape_side='left'): return asyncio.run(_escape_and_settle(escape_side)) -def test_the_tablet_goes_down_before_the_door_and_the_view_comes_back_after(): +def test_the_tablet_is_seen_going_down_before_the_door_is_pressed(): control, doors, controller, telemetry, state, tablet = _run('left') order = [call for call in controller.calls if call != 'select_camera'] - assert order[:4] == ['open_camera', 'close_camera', 'trigger_left_door', 'centre_view'], order + assert order[:3] == ['flip_tablet', 'flip_tablet', 'trigger_left_door'], order assert not control.is_open assert doors.closed['left'] and not doors.closed['right'] assert state.camera_open is False @@ -110,16 +122,19 @@ def test_the_escape_is_reported_as_its_own_release_and_door_cause(): def test_no_escape_is_started_without_a_dnp04_spike(): - control, doors, controller, telemetry, state, tablet = _raised(escape_side=None) - tasks.watch_the_tablet(tablet, control, doors, telemetry, state, now=2.0) + async def scenario(): + control, doors, controller, telemetry, state, tablet = await _raised(escape_side=None) + tasks.watch_the_tablet(tablet, control, doors, telemetry, state, now=2.0) + return control, doors, controller + control, doors, controller = asyncio.run(scenario()) assert control.is_open - assert 'close_camera' not in controller.calls + assert controller.calls == ['flip_tablet'] assert not any(doors.closed.values()) def test_a_second_escape_waits_for_the_first_lowering(): async def twice(): - control, doors, controller, telemetry, state, tablet = _raised('left') + control, doors, controller, telemetry, state, tablet = await _raised('left') first = control.escape(2.0, _noop) second = control.escape(2.1, _noop) await asyncio.sleep(0) @@ -140,7 +155,7 @@ def test_the_giant_fiber_leaves_the_door_alone_while_the_tablet_is_up(): assert not doors.refractory('left', now=1.0) -def test_a_door_is_slammed_once_and_reopened_when_the_hold_decays(): +def test_a_door_is_slammed_once_and_reopened_after_the_hold_decays_and_a_look_clears_it(): state = SensoryState() controller = RecordingController() doors = DoorControl(controller, ConnectomeTelemetry(), state) @@ -148,8 +163,9 @@ def test_a_door_is_slammed_once_and_reopened_when_the_hold_decays(): assert doors.on_giant_fiber('left', now=5.0) is False assert controller.calls == ['trigger_left_door'] + state.cleared_at['left'] = 9.0 for frame in range(200): - doors.release_decayed(now=6.0 + frame * 0.1, frame_elapsed=0.1) + doors.update(now=6.0 + frame * 0.1, frame_elapsed=0.1) assert controller.calls == ['trigger_left_door', 'open_left_door'] assert not doors.closed['left'] diff --git a/tests/test_tablet_map_detection.py b/tests/test_tablet_map_detection.py new file mode 100644 index 0000000..d707f2f --- /dev/null +++ b/tests/test_tablet_map_detection.py @@ -0,0 +1,70 @@ +import collections +import threading + +import cv2 +import numpy as np +import pytest + +from flynaf import config +from flynaf.env.tablet_map import count_camera_buttons, design_size, to_design +from flynaf.env.vision import FNAFVision + +WIDTH, HEIGHT = design_size() +BUTTON_SPOTS = [(160, 15), (140, 70), (40, 95), (120, 120), (300, 120), (20, 240), + (100, 240), (100, 285), (210, 240), (210, 285), (310, 190)] + + +def static(seed: int = 0) -> np.ndarray: + return np.random.default_rng(seed).integers(0, 120, size=(HEIGHT, WIDTH)).astype(np.uint8) + + +def camera_map(buttons=BUTTON_SPOTS, seed: int = 0) -> np.ndarray: + frame = static(seed) + for x, y in buttons: + cv2.rectangle(frame, (x, y), (x + 55, y + 33), 255, thickness=2) + cv2.rectangle(frame, (x + 3, y + 3), (x + 52, y + 30), 70, thickness=-1) + return frame + + +def office(seed: int = 0) -> np.ndarray: + frame = static(seed) // 2 + cv2.circle(frame, (300, 80), 18, 255, thickness=-1) + cv2.line(frame, (0, 200), (WIDTH, 220), 230, thickness=3) + return frame + + +@pytest.mark.parametrize('seed', range(4)) +def test_the_eleven_camera_buttons_are_counted_over_any_feed(seed): + assert count_camera_buttons(camera_map(seed=seed)) == len(BUTTON_SPOTS) + + +@pytest.mark.parametrize('seed', range(4)) +def test_an_office_with_bright_lamps_and_edges_holds_no_camera_button(seed): + assert count_camera_buttons(office(seed)) == 0 + + +def test_the_count_survives_a_window_of_another_size(): + double = cv2.resize(camera_map(), (WIDTH * 2, HEIGHT * 2), interpolation=cv2.INTER_NEAREST) + + assert count_camera_buttons(to_design(double)) == len(BUTTON_SPOTS) + + +def eye_on(counts: list) -> FNAFVision: + vision = FNAFVision.__new__(FNAFVision) + vision._buf_lock = threading.Lock() + vision._min_buttons = config.CAMERA_DETECTION.min_buttons + vision._map_buttons = collections.deque(counts, maxlen=config.CAMERA_DETECTION.agreeing_frames) + return vision + + +@pytest.mark.parametrize('counts, up, down', [ + ([11, 11, 11], True, False), + ([0, 0, 0], False, True), + ([0, 13, 11], False, False), + ([11, 11], False, False), +]) +def test_the_tablet_state_needs_every_recent_frame_to_agree(counts, up, down): + vision = eye_on(counts) + + assert vision.is_camera_up() is up + assert vision.is_camera_down() is down diff --git a/tests/test_vision_buffers.py b/tests/test_vision_buffers.py index 43e3718..dd7e73d 100644 --- a/tests/test_vision_buffers.py +++ b/tests/test_vision_buffers.py @@ -1,75 +1,50 @@ import collections import os import threading -import time import numpy as np from flynaf.env.vision import FNAFVision -def make_vision(calib_frames=5, calib_delay=0.01) -> FNAFVision: +def make_vision() -> FNAFVision: vision = FNAFVision.__new__(FNAFVision) - vision.mse_threshold = 1500.0 - vision._calib_frames = calib_frames - vision._calib_delay = calib_delay - vision.ref_left = None - vision.ref_right = None - vision._ref_camera_closed = None + vision.mse_threshold = 40.0 + vision.banks = {(side, closed): [] for side in ('left', 'right') for closed in (False, True)} + vision._distance_cache = {} vision.debug_dir = os.path.join(os.path.dirname(__file__), '_scratch_vision_debug') os.makedirs(vision.debug_dir, exist_ok=True) vision._left_buf = collections.deque(maxlen=5) vision._right_buf = collections.deque(maxlen=5) - vision._cam_buf = collections.deque(maxlen=1) + vision._window_bufs = {side: collections.deque(maxlen=5) for side in ('left', 'right')} + vision._map_buttons = collections.deque(maxlen=3) vision._buf_lock = threading.Lock() vision._threat_written_left = False vision._threat_written_right = False return vision -def test_clear_buffers_empties_all_three_queues(): +def test_clear_buffers_empties_every_queue(): vision = make_vision() vision._left_buf.append(np.zeros((2, 2), dtype=np.float32)) vision._right_buf.append(np.zeros((2, 2), dtype=np.float32)) - vision._cam_buf.append(np.zeros((2, 2), dtype=np.float32)) + for buf in vision._window_bufs.values(): + buf.append(np.zeros((2, 2), dtype=np.float32)) + vision._map_buttons.append(11) vision.clear_buffers() assert len(vision._left_buf) == 0 assert len(vision._right_buf) == 0 - assert len(vision._cam_buf) == 0 - - -def test_capture_camera_closed_reference_waits_for_frame_after_clear(): - vision = make_vision(calib_frames=20, calib_delay=0.01) - vision.clear_buffers() - - fresh_frame = np.full((2, 2), 42.0, dtype=np.float32) - - def populate_after_delay(): - time.sleep(0.03) - with vision._buf_lock: - vision._cam_buf.append(fresh_frame) - - threading.Thread(target=populate_after_delay).start() - vision.capture_camera_closed_reference() - - assert vision._ref_camera_closed is not None - assert np.array_equal(vision._ref_camera_closed, fresh_frame) - - -def test_capture_camera_closed_reference_gives_up_when_buffer_stays_empty(): - vision = make_vision(calib_frames=3, calib_delay=0.01) - - vision.capture_camera_closed_reference() - - assert vision._ref_camera_closed is None + assert all(len(buf) == 0 for buf in vision._window_bufs.values()) + assert len(vision._map_buttons) == 0 + assert not vision.is_camera_up() and not vision.is_camera_down() def test_sensory_rate_does_not_crash_on_empty_buffer_after_clear(): vision = make_vision() - vision.ref_left = np.zeros((2, 2), dtype=np.float32) - vision.ref_right = np.zeros((2, 2), dtype=np.float32) + for bank in vision.banks.values(): + bank.append(np.zeros((2, 2), dtype=np.float32)) vision.clear_buffers() assert vision.get_left_sensory_rate() == 0.0 @@ -77,8 +52,6 @@ def test_sensory_rate_does_not_crash_on_empty_buffer_after_clear(): if __name__ == '__main__': - test_clear_buffers_empties_all_three_queues() - test_capture_camera_closed_reference_waits_for_frame_after_clear() - test_capture_camera_closed_reference_gives_up_when_buffer_stays_empty() + test_clear_buffers_empties_every_queue() test_sensory_rate_does_not_crash_on_empty_buffer_after_clear() print('ok') diff --git a/tests/test_vision_reference_cache.py b/tests/test_vision_reference_cache.py index 3fabf85..eacccc1 100644 --- a/tests/test_vision_reference_cache.py +++ b/tests/test_vision_reference_cache.py @@ -1,3 +1,4 @@ +import os import tempfile import numpy as np @@ -9,55 +10,58 @@ def make_vision(reference_dir: str, l_bbox: int = 4, r_bbox: int = 6) -> FNAFVis vision = FNAFVision.__new__(FNAFVision) vision.l_bbox = l_bbox vision.r_bbox = r_bbox - vision.ref_left = None - vision.ref_right = None + vision.windows = {'left': (0, 0, 3, 5), 'right': (10, 10, 17, 12)} + vision.banks = {(side, closed): [] for side in ('left', 'right') for closed in (False, True)} + vision._distance_cache = {} vision.reference_dir = reference_dir return vision +def filled(writer: FNAFVision) -> FNAFVision: + for (side, closed), bank in writer.banks.items(): + shape = writer._bank_shape(side, closed) + base = 10.0 * closed + bank.extend(np.full(shape, base + level, dtype=np.float32) for level in (1.0, 2.0, 3.0)) + return writer + + def test_load_returns_false_when_no_cache_exists(): with tempfile.TemporaryDirectory() as tmp: vision = make_vision(tmp) - loaded = vision.load_reference_from_disk() - - assert loaded is False - assert vision.ref_left is None - assert vision.ref_right is None + assert vision.load_reference_from_disk() is False + assert all(not bank for bank in vision.banks.values()) -def test_save_then_load_round_trips_the_reference_frames(): +def test_save_then_load_round_trips_every_bank(): with tempfile.TemporaryDirectory() as tmp: - writer = make_vision(tmp) - writer.ref_left = np.arange(16, dtype=np.float32).reshape(4, 4) - writer.ref_right = np.arange(36, dtype=np.float32).reshape(6, 6) + writer = filled(make_vision(tmp)) writer.save_reference_to_disk() reader = make_vision(tmp) - loaded = reader.load_reference_from_disk() - assert loaded is True - assert np.array_equal(reader.ref_left, writer.ref_left) - assert np.array_equal(reader.ref_right, writer.ref_right) + assert reader.load_reference_from_disk() is True + for key, bank in writer.banks.items(): + assert len(reader.banks[key]) == len(bank) + for saved, loaded in zip(bank, reader.banks[key]): + assert np.array_equal(saved, loaded) def test_load_rejects_cache_with_mismatched_bbox_size(): with tempfile.TemporaryDirectory() as tmp: - writer = make_vision(tmp, l_bbox=4, r_bbox=6) - writer.ref_left = np.zeros((4, 4), dtype=np.float32) - writer.ref_right = np.zeros((6, 6), dtype=np.float32) - writer.save_reference_to_disk() + filled(make_vision(tmp, l_bbox=4, r_bbox=6)).save_reference_to_disk() reader = make_vision(tmp, l_bbox=10, r_bbox=6) - loaded = reader.load_reference_from_disk() - assert loaded is False - assert reader.ref_left is None - assert reader.ref_right is None + assert reader.load_reference_from_disk() is False + assert all(not bank for bank in reader.banks.values()) + +def test_a_cache_from_before_the_banks_forces_a_live_calibration(): + with tempfile.TemporaryDirectory() as tmp: + filled(make_vision(tmp)).save_reference_to_disk() + os.remove(os.path.join(tmp, 'banks.json')) + for name in ('ref_left.png', 'ref_right.png'): + open(os.path.join(tmp, name), 'wb').close() -if __name__ == '__main__': - test_load_returns_false_when_no_cache_exists() - test_save_then_load_round_trips_the_reference_frames() - test_load_rejects_cache_with_mismatched_bbox_size() - print('ok') + assert make_vision(tmp).load_reference_from_disk() is False From 5d7c35e37645f9d294a7b16d504a9bf2e06f20b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Artur=20Mendon=C3=A7a=20Arruda?= Date: Thu, 1 Oct 2026 22:05:35 -0300 Subject: [PATCH 12/14] feat(panel): move a narrow game aside for the full panel --- ...tarts-the-night-once-the-brain-is-ready.md | 18 +++++ src/flynaf/config.py | 1 + src/flynaf/env/desktop/__init__.py | 4 ++ src/flynaf/env/desktop/headless.py | 4 ++ src/flynaf/env/desktop/linux.py | 30 +++++--- src/flynaf/env/desktop/windows.py | 9 +++ src/flynaf/env/overlay.py | 14 ++++ src/flynaf/night/panel.py | 72 +++++++++++-------- tests/test_panel_docking.py | 39 ++++++++++ 9 files changed, 153 insertions(+), 38 deletions(-) create mode 100644 tests/test_panel_docking.py diff --git a/docs/adr/0021-the-launcher-starts-the-night-once-the-brain-is-ready.md b/docs/adr/0021-the-launcher-starts-the-night-once-the-brain-is-ready.md index eb90ab1..c213118 100644 --- a/docs/adr/0021-the-launcher-starts-the-night-once-the-brain-is-ready.md +++ b/docs/adr/0021-the-launcher-starts-the-night-once-the-brain-is-ready.md @@ -77,6 +77,22 @@ smaller than the display for a moment before it moves on. A windowed game does not lose its device when another window appears, and the panel can sit beside it. `GameLauncher.windowed` turns this off for an operator who accepts the risk. +A 1280x720 window centred on a 1920x1080 display leaves 320 pixels on each +side, under the 420 the full panel needs, so every night on such a display fell +back to the 280x150 overlay inside the game. When the game is windowed and +narrow enough, the panel now moves it to the left edge first and opens the full +panel in the space on its right, which `BrainView.dock_game` turns off. A fixed +position also keeps the eye references valid from one night to the next, since +the cached layout includes where the game sits. + +The panel window is Edge in app mode, and Edge draws its own title bar inside +the window, so removing the Windows caption left the bar in place. The page +reports its height to the panel server, the window is enlarged so the bar lies +above the requested rectangle, and a rounded window region cuts everything +outside it. The page height is rounded in device independent pixels and lands +up to two pixels off, so the region starts two pixels into the page and the +outline is drawn two pixels inside it. + The measurement of the menu points is the one step of setup that remains manual. It happens once per install rather than once per resolution, since the points are in the reference layout. @@ -112,6 +128,7 @@ browser tab open on this repository's page carries the name in its title, and it made the lookup report a running game, so the launcher skipped opening it and went straight to the brain. The title fallback now requires the whole title to be the game's, and the process name stays the primary test. + Being ready also has to mean that the brain has stopped starting up. Almost every recorded night opened with a door slam the eye did not cause. Of course it was weird, in seven of the eight traces long enough to tell, DNp01 fired within the first ten engine frames with both eye populations at zero input. The engine used to take its first step only once calibration had finished, from `reset_state`, which puts all 138,639 neurons at `v0` with no conductance. Stepped from there with only the background noise, the network recruits about 150 active neurons per frame up to about 3,000, excitation outruns the inhibition that later balances it, and over four measured starts the Giant Fiber fired inside that wave three times, always between frames 10 and 17, and never after frame 20. A real fly has no such moment, because its brain was running long before it was put in front of the hallways. The engine now steps without input while calibration runs, until the neurons active in the last ten frames are no more than 1.2 times those in the ten before, with a cap of 200 frames, and nothing in that warm-up reaches a motor path. The measured wave first meets the criterion around frame 30. Over six fresh starts the network settled after 29 to 35 frames and the Giant Fiber stayed silent through the 150 unstimulated frames that followed each one. The session report prints the warm-up, and `replay` settles the engine the same way before it feeds a recorded night. The window has to be measured in physical pixels too. Windows reports window rectangles scaled down by the display setting to any process that has not declared itself DPI aware, and the declaration used to happen as a side effect of the first `screen_size()` call. On a display at 150 percent, a run started with `--manual` measured the game at 213,131 sized 853x480 instead of 320,197 sized 1280x720 and placed the panel "beside" a game that actually extended under it, over the right eye, the right door and light buttons and part of the camera map. The Windows backend now declares DPI awareness when it is loaded, before anything reads a coordinate. @@ -125,3 +142,4 @@ The window has to be measured in physical pixels too. Windows reports window rec | 1.4 | Replaced the drawn head with the NeuroMechFly model in a night office scene | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-29 | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-29 | | 1.5 | Required the whole window title to match the game | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-30 | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-30 | | 1.6 | Warmed the network up before the night and declared DPI awareness before the first window measurement | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-10-01 | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-10-01 | +| 1.7 | Moved a narrow game window to the left edge for the full panel and cut the browser title bar off the panel | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-10-01 | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-10-01 | diff --git a/src/flynaf/config.py b/src/flynaf/config.py index 415cb10..95a80f8 100644 --- a/src/flynaf/config.py +++ b/src/flynaf/config.py @@ -130,6 +130,7 @@ class BrainView: patch_interval_sec: float = 0.3 patch_pixels: int = 72 beside_min_width: int = 420 + dock_game: bool = True corner_radius: int = 14 launch_browser: bool = True diff --git a/src/flynaf/env/desktop/__init__.py b/src/flynaf/env/desktop/__init__.py index ca329da..f912c3c 100644 --- a/src/flynaf/env/desktop/__init__.py +++ b/src/flynaf/env/desktop/__init__.py @@ -88,6 +88,10 @@ def apply_overlay_style(handle: int, x: int, y: int, w: int, h: int) -> bool: return _backend.apply_overlay_style(handle, x, y, w, h) +def move_window(handle: int, x: int, y: int) -> bool: + return _backend.move_window(handle, x, y) + + def outer_rect(handle: int): return _backend.outer_rect(handle) diff --git a/src/flynaf/env/desktop/headless.py b/src/flynaf/env/desktop/headless.py index 79234de..304f2ab 100644 --- a/src/flynaf/env/desktop/headless.py +++ b/src/flynaf/env/desktop/headless.py @@ -57,6 +57,10 @@ def apply_overlay_style(handle: int, x: int, y: int, w: int, h: int) -> bool: return False +def move_window(handle: int, x: int, y: int) -> bool: + return False + + def outer_rect(handle: int): return None diff --git a/src/flynaf/env/desktop/linux.py b/src/flynaf/env/desktop/linux.py index 43d86eb..2506ed5 100644 --- a/src/flynaf/env/desktop/linux.py +++ b/src/flynaf/env/desktop/linux.py @@ -6,7 +6,7 @@ import subprocess import time -from Xlib import X, display +from Xlib import X, display, error from Xlib.ext import shape, xtest _log = logging.getLogger(__name__) @@ -50,14 +50,14 @@ def _window(handle: int): return None try: return _dpy().create_resource_object('window', handle) - except Exception: + except error.XError: return None def _property(window, name: str, kind=X.AnyPropertyType): try: value = window.get_full_property(_atom(name), kind) - except Exception: + except error.XError: return None return value.value if value else None @@ -68,7 +68,7 @@ def _has_shape() -> bool: _SHAPE_CHECKED = True try: _HAS_SHAPE = _dpy().has_extension('SHAPE') - except Exception: + except error.XError: _HAS_SHAPE = False return _HAS_SHAPE @@ -93,7 +93,7 @@ def is_window(handle: int) -> bool: try: window.get_attributes() return True - except Exception: + except error.XError: return False @@ -146,7 +146,7 @@ def window_rect(handle: int): try: geometry = window.get_geometry() coords = window.translate_coords(_root(), 0, 0) - except Exception: + except error.XError: return None if geometry.width <= 0 or geometry.height <= 0: return None @@ -170,7 +170,7 @@ def _client_of(window, depth: int = 2): return None try: children = window.query_tree().children - except Exception: + except error.XError: return None for child in children: client = _client_of(child, depth - 1) @@ -185,7 +185,7 @@ def _mapped_clients() -> list: try: if frame.get_attributes().map_state != X.IsViewable: continue - except Exception: + except error.XError: continue client = _client_of(frame) if client is not None: @@ -199,7 +199,7 @@ def top_level_windows() -> list: return [int(handle) for handle in raw] try: return _mapped_clients() - except Exception: + except error.XError: return [] @@ -253,6 +253,18 @@ def apply_overlay_style(handle: int, x: int, y: int, w: int, h: int) -> bool: time.sleep(0.05) +def move_window(handle: int, x: int, y: int) -> bool: + window = _window(handle) + if window is None: + return False + try: + window.configure(x=x, y=y) + _dpy().sync() + except error.XError: + return False + return True + + def outer_rect(handle: int): return None diff --git a/src/flynaf/env/desktop/windows.py b/src/flynaf/env/desktop/windows.py index 16dde34..84c282d 100644 --- a/src/flynaf/env/desktop/windows.py +++ b/src/flynaf/env/desktop/windows.py @@ -18,6 +18,8 @@ _WS_EX_TRANSPARENT = 0x00000020 _WS_EX_NOACTIVATE = 0x08000000 _WS_EX_TOOLWINDOW = 0x00000080 +_SWP_NOSIZE = 0x0001 +_SWP_NOZORDER = 0x0004 _SWP_NOACTIVATE = 0x0010 _SWP_FRAMECHANGED = 0x0020 _SW_RESTORE = 9 @@ -227,6 +229,13 @@ def apply_overlay_style(handle: int, x: int, y: int, w: int, h: int) -> bool: return placed and on_top and not still_framed +def move_window(handle: int, x: int, y: int) -> bool: + if not handle: + return False + flags = _SWP_NOSIZE | _SWP_NOZORDER | _SWP_NOACTIVATE + return bool(_user32().SetWindowPos(handle, None, x, y, 0, 0, flags)) + + def outer_rect(handle: int): rect = ctypes.wintypes.RECT() if not handle or not ctypes.windll.user32.GetWindowRect(handle, ctypes.byref(rect)): diff --git a/src/flynaf/env/overlay.py b/src/flynaf/env/overlay.py index 95eff55..317ffe8 100644 --- a/src/flynaf/env/overlay.py +++ b/src/flynaf/env/overlay.py @@ -91,6 +91,20 @@ def beside_game_rect( return None +def dock_game_for_panel(screen_w: int, min_width: int, margin: int) -> bool: + handle = find_game_window() + client = desktop.window_rect(handle) if handle else None + outer = desktop.outer_rect(handle) if handle else None + if client is None or outer is None: + return False + if margin + client[2] + 2 * margin + min_width > screen_w: + return False + border = client[0] - outer[0] + if not desktop.move_window(handle, margin - border, outer[1]): + return False + return anchor_to_game() + + def ingame_overlay_rect(panel_w: int, panel_h: int, margin: int, step: int = 4) -> tuple | None: left, _top, right, bottom = game_bounds() forbidden = capture_regions() + motor_regions() diff --git a/src/flynaf/night/panel.py b/src/flynaf/night/panel.py index 4737b87..6dcded6 100644 --- a/src/flynaf/night/panel.py +++ b/src/flynaf/night/panel.py @@ -11,6 +11,7 @@ anchor_to_game, beside_game_rect, capture_regions, + dock_game_for_panel, find_browser, find_game_window, game_in_front, @@ -51,41 +52,54 @@ def restore_game_focus(game: int, panel: int, timeout_sec: float): ) -def _launch_brain_view_window(port: int, page_height) -> bool: +def _beside_the_game(screen_w: int, screen_h: int): params = config.BRAIN_VIEW - url = f'http://127.0.0.1:{port}/' - - screen_w, screen_h = screen_size() beside = beside_game_rect( screen_w, screen_h, params.beside_min_width, params.width, params.ingame_margin, ) - ingame = None if beside else ingame_overlay_rect( - params.ingame_width, params.ingame_height, params.ingame_margin, + if beside is not None or not params.dock_game: + return beside + if not dock_game_for_panel(screen_w, params.beside_min_width, params.ingame_margin): + return None + _log.info('game window moved to the left edge to make room for the panel') + return beside_game_rect( + screen_w, screen_h, params.beside_min_width, params.width, params.ingame_margin, ) + + +def _choose_panel_placement(url: str): + params = config.BRAIN_VIEW + screen_w, screen_h = screen_size() + beside = _beside_the_game(screen_w, screen_h) if beside is not None: - x, y, width, height = beside - page = f'{url}?corner={params.corner_radius}' - topmost = True - _log.info('brain view beside the game, panel %dx%d at %d,%d', width, height, x, y) - elif ingame is not None: - x, y = ingame - width, height = params.ingame_width, params.ingame_height - page = f'{url}?mini=1&corner={params.corner_radius}' - topmost = True - _log.info('brain view in-game overlay, panel %dx%d at %d,%d', width, height, x, y) - else: - position = pick_overlay_position( - screen_w, screen_h, params.width, params.height, - capture_regions() + motor_regions(), - ) - if position is None: - _log.warning('brain view has no screen area clear of capture and motor targets') - return False - x, y = position - width, height = params.width, params.height - page = url - topmost = False - _log.info('brain view side panel, panel %dx%d at %d,%d', width, height, x, y) + _log.info('brain view beside the game, panel %dx%d at %d,%d', beside[2], beside[3], *beside[:2]) + return (*beside, f'{url}?corner={params.corner_radius}', True) + + ingame = ingame_overlay_rect(params.ingame_width, params.ingame_height, params.ingame_margin) + if ingame is not None: + _log.info('brain view in-game overlay, panel %dx%d at %d,%d', + params.ingame_width, params.ingame_height, *ingame) + return (*ingame, params.ingame_width, params.ingame_height, + f'{url}?mini=1&corner={params.corner_radius}', True) + + position = pick_overlay_position( + screen_w, screen_h, params.width, params.height, capture_regions() + motor_regions(), + ) + if position is None: + _log.warning('brain view has no screen area clear of capture and motor targets') + return None + _log.info('brain view side panel, panel %dx%d at %d,%d', params.width, params.height, *position) + return (*position, params.width, params.height, url, False) + + +def _launch_brain_view_window(port: int, page_height) -> bool: + params = config.BRAIN_VIEW + url = f'http://127.0.0.1:{port}/' + + placement = _choose_panel_placement(url) + if placement is None: + return False + x, y, width, height, page, topmost = placement browser = find_browser() if browser is None: diff --git a/tests/test_panel_docking.py b/tests/test_panel_docking.py new file mode 100644 index 0000000..a59ba31 --- /dev/null +++ b/tests/test_panel_docking.py @@ -0,0 +1,39 @@ +from unittest.mock import patch + +import pytest + +from flynaf.env import overlay + +SCREEN_W = 1920 +MIN_WIDTH = 420 +MARGIN = 8 + + +def dock(client: tuple, outer: tuple): + moves = [] + with patch.object(overlay, 'find_game_window', return_value=7), \ + patch.object(overlay.desktop, 'window_rect', return_value=client), \ + patch.object(overlay.desktop, 'outer_rect', return_value=outer), \ + patch.object(overlay.desktop, 'move_window', + side_effect=lambda handle, x, y: moves.append((handle, x, y)) or True), \ + patch.object(overlay, 'anchor_to_game', return_value=True): + docked = overlay.dock_game_for_panel(SCREEN_W, MIN_WIDTH, MARGIN) + return docked, moves + + +def test_a_centred_game_moves_so_its_picture_starts_at_the_margin(): + docked, moves = dock(client=(208, 190, 1280, 720), outer=(197, 145, 1302, 776)) + + assert docked + assert moves == [(7, MARGIN - 11, 145)] + + +@pytest.mark.parametrize('client, outer', [ + ((0, 0, 1920, 1080), (0, 0, 1920, 1080)), + ((100, 100, 1600, 900), (89, 55, 1622, 956)), +]) +def test_a_game_too_wide_for_the_panel_stays_where_it_is(client, outer): + docked, moves = dock(client, outer) + + assert not docked + assert moves == [] From 3f02b2b17b29e867dc2b004c8617e3d172a94e0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Artur=20Mendon=C3=A7a=20Arruda?= Date: Thu, 1 Oct 2026 22:09:47 -0300 Subject: [PATCH 13/14] docs: describe the new eye, tablet, doors and panel and drop the roadmap --- README.md | 24 ++++++++++++++++-------- docs/roadmap.md | 34 ---------------------------------- 2 files changed, 16 insertions(+), 42 deletions(-) delete mode 100644 docs/roadmap.md diff --git a/README.md b/README.md index 87e0113..6fb2b37 100644 --- a/README.md +++ b/README.md @@ -15,13 +15,13 @@ https://www.youtube.com/watch?v=4UNPlA-YJtw. Four behaviours come out of the network, and each one is a different pathway. -**Slamming a door.** The hallway is captured while the light is on and compared against a reference of that same hallway empty. A difference above threshold drives the corresponding eye cluster at full rate, every LPLC2 and LC4 neuron on that side, that excitation propagates through the real connectivity, and when DNp01, the Giant Fiber, crosses its own firing threshold the door closes. Measured on this machine, the neuron needs seven to nine engine frames of sustained input to answer, which is why the light is held for twelve. +**Slamming a door.** The hallway is captured while the light is on, which the fly confirms by reading the light button on screen, and compared against the views of that hallway recorded empty at midnight, lit, flickering and dark. A frame that is far from all of them drives the corresponding eye cluster at full rate, every LPLC2 and LC4 neuron on that side, that excitation propagates through the real connectivity, and when DNp01, the Giant Fiber, crosses its own firing threshold the door closes. Measured on this machine, the neuron needs seven to nine engine frames of sustained input to answer, which is why the light is held for twelve. The door stays shut while the fly can still see the threat. With the door closed the light still reaches the window, where Bonnie casts a shadow and Chica stands, and the eye reads that window on its own against the lit view of the empty window behind a closed door, so a dark window never passes for an empty one. A threat still there fires DNp01 again and renews the hold, and the door only reopens once the defensive state has decayed and the latest look at that side, taken with the light confirmed on, came back empty (ADR 0011, experiment 14). **Checking a hallway.** The difference in membrane potential between the left and right eye populations is a slow spontaneous signal the network produces on its own, with a lag-1 autocorrelation of 0.97. It is accumulated to a bound together with what the last look revealed and a per-side habituation term, and whichever side wins gets looked at. There is one starvation guard, a clock that forces a look at a hallway left unwatched for thirty seconds, and every look it causes is counted separately in the session report so its share stays visible. **Raising the monitor.** DNp09's own subthreshold membrane potential is accumulated the same way. A real spike raises the tablet outright, and the tablet comes down when that drive fades rather than when a timer expires. -Raising the tablet also drives the GABAergic inhibitor clusters at full rate, which silences the Giant Fiber, so no door reflex can fire through the tablet, exactly as no door button can be pressed in the game. +Raising the tablet also drives the GABAergic inhibitor clusters at full rate, which silences the Giant Fiber, so no door reflex can fire through the tablet, exactly as no door button can be pressed in the game. Whether the tablet is up is read off the screen, by counting the eleven camera buttons of the map, and the fly believes a raise or a lowering only once the map has appeared or gone (ADR 0015). **Watching for Foxy.** The tablet always comes up on Pirate Cove, CAM 1C. What differs from the cove as it looked at the start of the night drives LC9 and LC31a, the lobula columns that feed DNp09 and are used by flies to pursue a moving object. When DNp09 fires, the gaze follows to the west hall, CAM 2A. There the size of whatever differs drives LPLC2 and its growth drives LC4, and DNp04, a looming escape neuron the inhibitors barely reach, fires on it. The fly then drops the tablet and closes the left door. Measured on the engine, each channel wakes its own descending neuron and never the other, and the Giant Fiber stays silent through both (ADR 0026, experiment 13). @@ -138,8 +138,9 @@ each region actually reads. Each one runs as a module, for example - `src/flynaf/scripts/select_roi.py` takes a screenshot and lets you drag the two hallway capture boxes. - `src/flynaf/scripts/vision_calibrator.py` writes the measured values into `logs/`. -- `src/flynaf/scripts/debug_vision.py` and `src/flynaf/scripts/debug_camera.py` show live what each capture region is reading. +- `src/flynaf/scripts/debug_vision.py` shows live what each eye region is reading, and `src/flynaf/scripts/debug_camera.py` prints how many camera buttons the map region holds and whether that counts as the tablet being up. - `src/flynaf/scripts/debug_reflex.py` shows the live contrast of both hallways against the threshold, and says whether a door would slam. Run it with a light held on, because a reading taken with the light off means nothing. +- `src/flynaf/scripts/measure_door_window.py`, given `left` or `right`, closes that door, lights the window and prints how far it strays from the closed-door reference. Run it once with the hallway empty and once with Bonnie or Chica outside, and set `closed_door_mse_threshold` between the two readings. - `src/flynaf/scripts/debug_tablet.py`, given `1C` or `2A`, records one camera with nothing in view and then prints the live drive it would send into the brain. That is how the spans in `TabletVision` are fitted, and how the CAM 2A button position, inferred from CAM 4B, is confirmed. The camera references are not cached. At the start of every night the fly raises the tablet once, records a few seconds of CAM 1C and CAM 2A while the curtain is closed and the hall is empty, and lowers it again, which also stalls Foxy while it happens. @@ -190,16 +191,18 @@ night. `--manual`, or the button at the bottom of the launcher, keeps the old flow. The brain loads, the activity panel opens over the office, and a ten second countdown starts. Go to the game and start a night inside that window. In every -case the fly then centres the view, captures its office reference and takes -over. +case the fly then makes sure the tablet is down and takes over. Before that, +while the game loads, the network steps on its own until its activity has +settled, so the night does not start inside the recruitment wave of a brain +that was just switched on (ADR 0021). -The first run after a calibration change performs live calibration, which turns each hallway light on in turn to record what an empty hallway looks like. **Both hallways have to be empty at that moment**, otherwise an animatronic gets recorded as the normal state and the fly stays blind for the rest of the night. +The first run after a calibration change performs live calibration, which turns each hallway light on in turn to record what an empty hallway looks like, then closes that door for a moment to record the empty window behind it. **Both hallways have to be empty at that moment**, otherwise an animatronic gets recorded as the normal state and the fly stays blind for the rest of the night. Every run writes a report to `logs/session_telemetry_*.txt` covering who decided each look, what the eye measured against the threshold, how long the eye drove the cluster, and how much of the night the inhibitors were active. One real report is kept at `docs/example-session-report.txt` so the format and the numbers can be read without running anything. ## The live activity panel -When the game runs in a window, the panel sits beside it at the height of the game, so one screen recording holds both. It shows all 138,639 somata in their real anatomical positions, lit as they fire, between two readouts of the same clock. Above the brain is what the fly receives, the patch of each hallway, how far it differs from the reference against the threshold, and whether that input is driving the eye cluster, greyed out while the light on that side is off. Below it is a 20 second timeline of the input, the eye cluster spikes, DNp01, the door and the tablet. +When the game runs in a window, the panel sits beside it at the height of the game, so one screen recording holds both. A window centred on the screen with no room left beside it is moved to the left edge first. It shows all 138,639 somata in their real anatomical positions, lit as they fire, between two readouts of the same clock. Above the brain is what the fly receives, the patch of each hallway, how far it differs from the reference against the threshold, and whether that input is driving the eye cluster, greyed out while the light on that side is off. Below it is a 20 second timeline of the input, the eye cluster spikes, DNp01, the door and the tablet. When DNp01 fires, the panel traces the spike backwards through the synapses whose presynaptic neuron fired just before it, and replays that trace over the real neurons 150 times slower than simulated time, with a caption giving its depth in frames and simulated milliseconds and whether it reaches the eye cluster. Nothing on the panel is triggered by the game. Each element is a variable the simulation already holds, and ADR 0022 explains what the trace does and does not show. @@ -269,7 +272,10 @@ and why the game is measured only once it is in front, and ADR 0022 covers the panel and the rule that only simulation variables drive it. ADR 0025 covers how populations are named by cell type rather than pasted as ids, and ADR 0026 and experiment 13 cover the tablet, Foxy and why DNp04 rather than the Giant Fiber -answers through it. ADR 0027 covers the `flynaf` package and the test pipeline. +answers through it. ADR 0027 covers the `flynaf` package and the test pipeline, +and ADR 0028 covers the single clock every time step of a night is read from. +ADR 0011 and experiment 14 cover why the fly once reopened the door into Bonnie +and how the window behind a closed door is read now. ## Licensing and credits @@ -305,3 +311,5 @@ Five Nights at Freddy's is by Scott Cawthon and is not affiliated with this proj | 1.15 | Pointed to the night package that main now wires | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-30 | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-30 | | 1.16 | Described the tablet watch on Foxy, the cell type populations and the new scripts, and corrected which tests are slow | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-30 | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-30 | | 1.17 | Moved the paths and commands to the flynaf package, described the pytest and CI pipeline and the connectome marker | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-10-01 | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-10-01 | +| 1.18 | Described the door that reopens after a clearing look, the closed-door calibration, the tablet read from the camera map, the warm-up and the bank of hallway views | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-10-01 | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-10-01 | +| 1.19 | Described the window read behind a closed door, the lit look a door needs to reopen and the game moved aside for the panel | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-10-01 | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-10-01 | diff --git a/docs/roadmap.md b/docs/roadmap.md deleted file mode 100644 index 0214b88..0000000 --- a/docs/roadmap.md +++ /dev/null @@ -1,34 +0,0 @@ -# FNAF Fly Brain Connectome: Project Roadmap - -This document outlines the strategic phases for the cybernetic integration of the *Drosophila melanogaster* connectome with the interactive environment of Five Nights at Freddy's. - -## Phase 1: Proof of Concept & Infrastructure -**Status:** Completed -**Objective:** Validate the technical bridge between the game environment, computer vision, biological simulation, and the operating system. -**Scope:** -- Real-time screen capture via OpenCV. -- Tensor injection into the PyTorch simulation engine. -- Low-level OS mouse control via Win32 ctypes. -- Implemented a temporary biological bypass (ADR 0003) to inject visual stimuli directly into the motor cortex due to signal decay in default sensory pathways. - -## Phase 2: Biological Graph Routing -**Status:** Pending -**Objective:** Replace the artificial bypass with anatomically accurate neural pathways. -**Scope:** -- Parse the 100MB sparse connectivity matrix. -- Implement Dijkstra's algorithm to compute the shortest and most excitatory paths from sensory clusters to motor output nodes. -- Map the four primary game inputs to the discovered biological sensory nodes, ensuring stimuli propagate naturally through the connectome to trigger the required motor reflexes. - -## Phase 3: Autonomous Cybernetic Survival -**Status:** Pending -**Objective:** Establish full autonomy driven by biological constraints and game state translation. -**Scope:** -- Inject constant low-level stimuli to induce exploratory behavior (randomized viewport checking). -- Implement computer vision tracking for in-game power levels. -- Translate battery depletion into simulated biological fatigue by dampening the firing rates dynamically. -- Prove embodied connectomics by forcing the biological network to manage digital resources under stress. - -| Version | Description | Author(s) | Date | Reviewer(s) | Review Date | -|---------|-------------|-----------|------|-------------|-------------| -| 1.0 | Initial version | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-02 | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-02 | -| 1.1 | ADR reference style aligned with the rest of the documentation | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-18 | [Artur Mendonça Arruda](https://github.com/ArtyMend07) | 2026-09-18 | From db0753b64b32c64621c5e52650989975be280db9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Artur=20Mendon=C3=A7a=20Arruda?= Date: Thu, 1 Oct 2026 22:16:29 -0300 Subject: [PATCH 14/14] chore: release 0.3.0 --- pyproject.toml | 2 +- uv.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 30065ca..0823456 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "fly-naf" -version = "0.1.0" +version = "0.3.0" description = "A whole-brain Drosophila connectome simulation that plays Five Nights at Freddy's 1" readme = "README.md" license = "GPL-3.0-or-later" diff --git a/uv.lock b/uv.lock index 9d8c0d8..5cb245b 100644 --- a/uv.lock +++ b/uv.lock @@ -529,7 +529,7 @@ wheels = [ [[package]] name = "fly-naf" -version = "0.1.0" +version = "0.3.0" source = { editable = "." } dependencies = [ { name = "caveclient" },