From 4f8b9bccfedfd16920507314fa4f2bce8c68cd49 Mon Sep 17 00:00:00 2001 From: Roddy Date: Tue, 4 Aug 2026 16:06:49 +0200 Subject: [PATCH 1/3] fix(analytics): compare each EI player entry against its own fight slice Elite Insights does not emit one entry per player. It emits one per contiguous stretch of squad membership: a player who leaves and rejoins the squad, or who is out of it before the recorder picks them up, appears several times under the same account and the same instance ID, with adjacent non-overlapping firstAware/lastAware windows and a different `group` on each. Every counter on an entry covers only that stretch. On `20260424-204954`, 34 entries cover 23 accounts, with `empiria.8961` split three ways at 3..11321 / 11322..137166 / 137167..254724. arcdps has no such notion -- one agent record spans the whole fight -- so the comparison was measuring each of those entries against the player's whole-fight totals. Every counter on every duplicated entry came out high, which is exactly why the per-target buckets dominated the report while `statsAll` looked healthy: the per-target arrays are per-entry too, so a 3-way split inflated 69 target rows at once. Each entry is now compared against the matching slice of the event stream, and the difference key carries the slice so entries under one account stop overwriting each other. The correlation is exact: the five logs with duplicated entries were the five worst in the corpus, and logs with no duplicates were already near-clean. Corpus: 4 503 -> 1 140 differences over 35 logs. 20260424-204954 2 202 -> 182 20260224-233019 674 -> 72 20260526-202841 503 -> 142 20260412-220632 280 -> 24 20260125-194936 198 -> 62 Buff uptimes and consumables are still computed whole-fight and need the same treatment; they are now the largest remaining buckets. Signed-off-by: Roddy Co-Authored-By: Claude Opus 5 --- .../src/gw2_analytics/ei_compare.py | 126 +++++++++++++++--- 1 file changed, 107 insertions(+), 19 deletions(-) diff --git a/libs/gw2_analytics/src/gw2_analytics/ei_compare.py b/libs/gw2_analytics/src/gw2_analytics/ei_compare.py index ee9aefc2..a809a2ca 100644 --- a/libs/gw2_analytics/src/gw2_analytics/ei_compare.py +++ b/libs/gw2_analytics/src/gw2_analytics/ei_compare.py @@ -2,7 +2,7 @@ from __future__ import annotations -from collections import defaultdict +from collections import Counter, defaultdict from collections.abc import Sequence from typing import Any @@ -193,6 +193,28 @@ def _skill_stats(events: list[DamageEvent]) -> dict[int, dict[str, int]]: return result +def _slice_bounds(player: dict[str, Any], origin: int) -> tuple[int, int]: + """Absolute ms bounds of the fight slice one EI player entry covers. + + Elite Insights does not emit one entry per player: it emits one per + *contiguous stretch of squad membership*. A player who leaves and + rejoins the squad -- or who is simply out of it before the recorder + picks them up -- appears several times under the same account and the + same instance ID, with adjacent, non-overlapping firstAware/lastAware + windows and a different ``group`` on each. Every counter on an entry + covers only that stretch. + + arcdps has no such notion: one agent record spans the whole fight. So + each EI entry has to be compared against the matching slice of our + event stream rather than against the player's whole-fight totals. + """ + first = player.get("firstAware") + last = player.get("lastAware") + if not isinstance(first, int) or not isinstance(last, int) or last < first: + return (-(1 << 62), 1 << 62) + return (origin + first, origin + last) + + def compare_elite_insights( # noqa: PLR0912, PLR0915 fight: Fight, expected: dict[str, object], @@ -258,6 +280,8 @@ def compare_elite_insights( # noqa: PLR0912, PLR0915 outcome_events_by_target: dict[int, list[CombatOutcomeEvent]] = defaultdict(list) block_count_by_source: dict[int, int] = defaultdict(int) dodge_count_by_source: dict[int, int] = defaultdict(int) + block_events_by_source: dict[int, list[BlockEvent]] = defaultdict(list) + dodge_events_by_source: dict[int, list[DodgeEvent]] = defaultdict(list) downed_buff_times_by_target: dict[int, set[int]] = defaultdict(set) for down_event in down_events: down_events_by_source[down_event.source_agent_id].append(down_event) @@ -272,8 +296,10 @@ def compare_elite_insights( # noqa: PLR0912, PLR0915 for indexed_event in event_list: if isinstance(indexed_event, BlockEvent): block_count_by_source[indexed_event.source_agent_id] += 1 + block_events_by_source[indexed_event.source_agent_id].append(indexed_event) elif isinstance(indexed_event, DodgeEvent): dodge_count_by_source[indexed_event.source_agent_id] += 1 + dodge_events_by_source[indexed_event.source_agent_id].append(indexed_event) elif ( isinstance(indexed_event, BoonApplyEvent) and indexed_event.kind == "apply" @@ -336,6 +362,11 @@ def indexed_events(index: dict[int, list[Any]], agent_ids: set[int]) -> list[Any isinstance(event, BoonApplyEvent) and event.kind == "apply" and event.skill_id == 770 for event in event_list ) + account_entry_count: Counter[str] = Counter( + entry["account"] + for entry in expected_players + if isinstance(entry, dict) and isinstance(entry.get("account"), str) + ) compared_players: dict[str, object] = {} for player in expected_players: @@ -350,9 +381,22 @@ def indexed_events(index: dict[int, list[Any]], agent_ids: set[int]) -> list[Any compared_players[account] = None differences[f"players[{account}]"] = {"expected": "present", "actual": None} continue - prefix = f"players[{account}]" + slice_lo, slice_hi = _slice_bounds(player, origin) + # Several EI entries can share an account; key the report by the + # slice so their differences do not overwrite each other. + prefix = ( + f"players[{account}]" + if account_entry_count[account] < 2 + else f"players[{account}@{player.get('firstAware')}]" + ) anonymous = agent.account_name is None agent_ids = agent_ids_by_instance.get(agent.instance_id, {agent.id}) + + whole_fight_slice = slice_lo <= origin and slice_hi >= origin + duration_ms + + def in_slice(event: Event, _lo: int = slice_lo, _hi: int = slice_hi) -> bool: + return _lo <= event.time_ms <= _hi + # An anonymized enemy player carries no account name, and arcdps # has replaced their character name with the *localized* spec # string. EI labels them " pl-", so the @@ -380,10 +424,14 @@ def indexed_events(index: dict[int, list[Any]], agent_ids: set[int]) -> list[Any source_damage = [ event for event in damage_events - if event.source_agent_id in agent_ids and event.source_agent_id != event.target_agent_id + if event.source_agent_id in agent_ids + and event.source_agent_id != event.target_agent_id + and in_slice(event) ] actor_damage = [event for event in source_damage if event.src_master_instid == 0] - source_cc = [event for event in cc_events if event.source_agent_id in agent_ids] + source_cc = [ + event for event in cc_events if event.source_agent_id in agent_ids and in_slice(event) + ] dps_all = player.get("dpsAll") if isinstance(dps_all, list) and dps_all and isinstance(dps_all[0], dict): @@ -401,12 +449,20 @@ def indexed_events(index: dict[int, list[Any]], agent_ids: set[int]) -> list[Any defenses = player.get("defenses") if isinstance(defenses, list) and defenses and isinstance(defenses[0], dict): - taken = [event for event in damage_events if event.target_agent_id in agent_ids] - downed = indexed_events(down_events_by_source, agent_ids) + taken = [ + event + for event in damage_events + if event.target_agent_id in agent_ids and in_slice(event) + ] + downed = [ + event + for event in indexed_events(down_events_by_source, agent_ids) + if in_slice(event) + ] outcome_downs = { event.time_ms for event in indexed_events(outcome_events_by_target, agent_ids) - if event.outcome == "downed" + if event.outcome == "downed" and in_slice(event) } defense_values = { "damageTaken": sum(event.damage for event in taken), @@ -417,15 +473,22 @@ def indexed_events(index: dict[int, list[Any]], agent_ids: set[int]) -> list[Any "powerDamageTaken": sum( event.damage - _condition_damage(event, condition_skill_ids) for event in taken ), - "blockedCount": block_count_by_source[agent.id], - "evadedCount": dodge_count_by_source[agent.id], + "blockedCount": block_count_by_source[agent.id] + if whole_fight_slice + else sum(1 for event in block_events_by_source[agent.id] if in_slice(event)), + "evadedCount": dodge_count_by_source[agent.id] + if whole_fight_slice + else sum(1 for event in dodge_events_by_source[agent.id] if in_slice(event)), "downCount": ( len(outcome_downs) if outcome_downs and not anonymous else len( - set().union( - *(downed_buff_times_by_target[agent_id] for agent_id in agent_ids) - ) + { + time_ms + for agent_id in agent_ids + for time_ms in downed_buff_times_by_target[agent_id] + if slice_lo <= time_ms <= slice_hi + } ) ) or ( @@ -436,7 +499,11 @@ def indexed_events(index: dict[int, list[Any]], agent_ids: set[int]) -> list[Any else 0 ), "downDuration": sum(event.downtime_ms for event in downed), - "deadCount": len(indexed_events(death_events_by_source, agent_ids)), + "deadCount": sum( + 1 + for event in indexed_events(death_events_by_source, agent_ids) + if in_slice(event) + ), } values["defenses"] = defense_values _compare_fields(f"{prefix}.defenses", defenses[0], defense_values, differences) @@ -515,6 +582,7 @@ def indexed_events(index: dict[int, list[Any]], agent_ids: set[int]) -> list[Any (cast.skill_id, cast.time_ms, cast.duration_ms) for cast in rotation if cast.source_agent_id == agent.id + and slice_lo <= cast.time_ms + origin <= slice_hi ), key=lambda item: (item[1], item[0]), ) @@ -535,7 +603,7 @@ def indexed_events(index: dict[int, list[Any]], agent_ids: set[int]) -> list[Any actual_consumables = sorted( (buff.skill_id, buff.time_ms, buff.duration_ms, buff.stacks) for buff in extract_initial_buffs(event_list, origin, ids) - if buff.agent_id == agent.id + if buff.agent_id == agent.id and slice_lo <= buff.time_ms + origin <= slice_hi ) expected_consumables = sorted( (item["id"], item["time"], item["duration"], item["stack"]) @@ -577,12 +645,32 @@ def indexed_events(index: dict[int, list[Any]], agent_ids: set[int]) -> list[Any target_cc = [event for event in source_cc if event.target_agent_id in target_ids] target_down_rows = DownContributionAggregator().aggregate( actor_target_damage, - indexed_events(down_events_by_source, target_ids), - indexed_events(death_events_by_source, target_ids), + [ + event + for event in indexed_events(down_events_by_source, target_ids) + if in_slice(event) + ], + [ + event + for event in indexed_events(death_events_by_source, target_ids) + if in_slice(event) + ], duration_ms / 1000, - health_events=indexed_events(health_events_by_source, target_ids), - up_events=indexed_events(up_events_by_source, target_ids), - outcome_events=indexed_events(outcome_events_by_target, target_ids), + health_events=[ + event + for event in indexed_events(health_events_by_source, target_ids) + if in_slice(event) + ], + up_events=[ + event + for event in indexed_events(up_events_by_source, target_ids) + if in_slice(event) + ], + outcome_events=[ + event + for event in indexed_events(outcome_events_by_target, target_ids) + if in_slice(event) + ], cc_events=target_cc, ) target_down_by_source = {row.source_agent_id: row for row in target_down_rows} From f6e4f0b758b980a7c6f6f63d184a5a7b404e6861 Mon Sep 17 00:00:00 2001 From: Roddy Date: Tue, 4 Aug 2026 16:12:25 +0200 Subject: [PATCH 2/3] docs(analytics): record that EI keeps buff uptimes whole-fight on split entries Follow-up to the slice fix. The obvious next step was to window buff uptimes the same way as the damage counters, and it is wrong: EI reports the same whole-fight buffUptimes on every entry of a split account. Measured on the four logs that have splits, restricting uptimes to the slice takes buff differences from 225 to 913. Two dead ends are worth recording so they are not retried: - subtracting two BuffStateTracker readings does not give the stack-time inside a window; the tracker only moves forward, so both readings taken after the full feed return the same final total; - replaying the tracker per window computes the window correctly and still diverges, because the target itself is whole-fight. Signed-off-by: Roddy Co-Authored-By: Claude Opus 5 --- libs/gw2_analytics/src/gw2_analytics/ei_compare.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/libs/gw2_analytics/src/gw2_analytics/ei_compare.py b/libs/gw2_analytics/src/gw2_analytics/ei_compare.py index a809a2ca..115cfd68 100644 --- a/libs/gw2_analytics/src/gw2_analytics/ei_compare.py +++ b/libs/gw2_analytics/src/gw2_analytics/ei_compare.py @@ -334,10 +334,8 @@ def indexed_events(index: dict[int, list[Any]], agent_ids: set[int]) -> list[Any else set() ) - tracker = BuffStateTracker( - start_time_ms=origin, - healing_by_agent={agent.id: agent.healing for agent in fight.agents}, - ) + healing_by_agent = {agent.id: agent.healing for agent in fight.agents} + tracker = BuffStateTracker(start_time_ms=origin, healing_by_agent=healing_by_agent) for tracked_event in event_list: if isinstance(tracked_event, (BoonApplyEvent, BuffApplyEvent, BuffExtensionEvent)): tracker.process(tracked_event) @@ -524,6 +522,11 @@ def in_slice(event: Event, _lo: int = slice_lo, _hi: int = slice_hi) -> bool: expected_buffs = player.get("buffUptimes") if isinstance(expected_buffs, list): + # Uptimes stay whole-fight even on a sliced entry: EI reports + # the same buffUptimes on every entry of a split account, so + # restricting them to the slice measurably diverges (verified on + # the corpus: windowing them takes buff differences from 225 to + # 913 across the four logs that have splits). uptime = dict.fromkeys(TRACKED_BUFFS, 0.0) for alias_id in agent_ids: alias_uptime = tracker.compute_player_uptimes(alias_id, duration_ms) From a4dd6c43bdae7d4db5fabcb926f7a6fc291fe11b Mon Sep 17 00:00:00 2001 From: Roddy Date: Tue, 4 Aug 2026 16:15:41 +0200 Subject: [PATCH 3/3] fix(analytics): stop slicing consumables out of split entries Consumables are pre-fight applications, and EI repeats the same list on every entry of a split account. Restricting them to the entry's window dropped them whenever a player's firstAware landed after the food or utility went up, which the slice change had just made possible. Corpus: 1 140 -> 1 116 differences; consumables 82 -> 58. Also tried and rejected: shifting our consumable timestamps by +1 ms. 39 mismatching entries are short by exactly 1, but most entries already agree, so the blanket shift takes consumables from 39 to 95 on the four logs that carry them. The remaining gap is a mix of that offset and entries we never emit, not one rule. Signed-off-by: Roddy Co-Authored-By: Claude Opus 5 --- libs/gw2_analytics/src/gw2_analytics/ei_compare.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/libs/gw2_analytics/src/gw2_analytics/ei_compare.py b/libs/gw2_analytics/src/gw2_analytics/ei_compare.py index 115cfd68..b89e90ce 100644 --- a/libs/gw2_analytics/src/gw2_analytics/ei_compare.py +++ b/libs/gw2_analytics/src/gw2_analytics/ei_compare.py @@ -606,7 +606,11 @@ def in_slice(event: Event, _lo: int = slice_lo, _hi: int = slice_hi) -> bool: actual_consumables = sorted( (buff.skill_id, buff.time_ms, buff.duration_ms, buff.stacks) for buff in extract_initial_buffs(event_list, origin, ids) - if buff.agent_id == agent.id and slice_lo <= buff.time_ms + origin <= slice_hi + # Not sliced: consumables are pre-fight applications that EI + # repeats on every entry of a split account, and a player whose + # firstAware is later than the application would otherwise lose + # them entirely. + if buff.agent_id == agent.id ) expected_consumables = sorted( (item["id"], item["time"], item["duration"], item["stack"])