Skip to content

Commit ca016c7

Browse files
eimexdeveimexdev
committed
feat: add Hermes voice notes and context usage
1 parent ee6764b commit ca016c7

19 files changed

Lines changed: 1165 additions & 61 deletions

‎CONTEXT.md‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,17 +97,41 @@ A blocking Hermes request for information needed to continue an agent run.
9797
_Avoid_: Approval, follow-up message
9898

9999
**Voice note**:
100-
A recorded user message delivered to Hermes as speech for automatic transcription.
100+
A recorded user message delivered to Hermes as speech for automatic transcription, optionally with typed accompaniment.
101101
_Avoid_: Audio file, voice channel
102102

103+
**Voice recording**:
104+
An in-progress microphone capture that may be stopped into a voice draft or stopped and delivered immediately as a voice note.
105+
_Avoid_: Voice draft, live voice message
106+
107+
**Recording bar**:
108+
The persistent control surface for a voice recording when the user navigates away from its originating thread.
109+
_Avoid_: Composer recorder, voice player
110+
103111
**Voice draft**:
104112
A completed but unsent voice recording available for replay, discard, or delivery.
105113
_Avoid_: Paused recording, attachment
106114

115+
**Interrupted voice draft**:
116+
A recoverable voice draft created when capture ends unexpectedly after recording usable audio.
117+
_Avoid_: Failed voice note, corrupted recording
118+
107119
**Voice transcript**:
108120
The Hermes-produced text associated with a specific voice note.
109121
_Avoid_: Assistant reply, caption
110122

123+
**Typed accompaniment**:
124+
Optional user-authored text delivered with a voice note to frame, qualify, or instruct Hermes about the recording.
125+
_Avoid_: Transcript, caption
126+
127+
**Context footprint**:
128+
The latest real Hermes-reported prompt-token occupancy for a thread relative to its absolute effective model context length. T3 Agent hides the context-window meter until that first trustworthy reading exists; Hermes's automatic-compaction threshold is explanatory metadata rather than the ring's denominator. It is distinct from cumulative tokens processed across the session, especially after compaction.
129+
_Avoid_: Session token total, billing usage
130+
131+
**Recording stop control**:
132+
The explicit control that ends microphone capture and creates a voice draft, presented with the same red-square visual language as T3 Code's active-turn stop control.
133+
_Avoid_: Pause, cancel recording
134+
111135
## Automation and remote access
112136

113137
**Cron execution session**:

‎apps/server/src/attachmentStore.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,8 @@ export function attachmentRelativePath(attachment: ChatAttachment): string {
6363
});
6464
return `${attachment.id}${extension}`;
6565
}
66+
case "audio":
67+
return `${attachment.id}.bin`;
6668
}
6769
}
6870

‎apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -636,6 +636,20 @@ export function runtimeEventToActivities(
636636
}
637637

638638
case "item.updated": {
639+
if (event.payload.itemType === "user_message") {
640+
return [
641+
{
642+
id: event.eventId,
643+
createdAt: event.createdAt,
644+
tone: event.payload.status === "failed" ? "error" : "info",
645+
kind: "voice-transcription.updated",
646+
summary: event.payload.title ?? "Voice transcription updated",
647+
payload: event.payload.data ?? {},
648+
turnId: toTurnId(event.turnId) ?? null,
649+
...maybeSequence,
650+
},
651+
];
652+
}
639653
if (!isToolLifecycleItemType(event.payload.itemType)) {
640654
return [];
641655
}

‎apps/server/src/orchestration/Normalizer.ts‎

Lines changed: 29 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import {
77
type IsoDateTime,
88
type OrchestrationCommand,
99
OrchestrationDispatchCommandError,
10+
PROVIDER_SEND_TURN_MAX_AUDIO_BYTES,
1011
PROVIDER_SEND_TURN_MAX_IMAGE_BYTES,
1112
} from "@t3tools/contracts";
1213

@@ -109,16 +110,21 @@ export const normalizeDispatchCommand = (command: ClientOrchestrationCommand) =>
109110
(attachment) =>
110111
Effect.gen(function* () {
111112
const parsed = parseBase64DataUrl(attachment.dataUrl);
112-
if (!parsed || !parsed.mimeType.startsWith("image/")) {
113+
const expectedMimePrefix = attachment.type === "audio" ? "audio/" : "image/";
114+
if (!parsed || !parsed.mimeType.startsWith(expectedMimePrefix)) {
113115
return yield* new OrchestrationDispatchCommandError({
114-
message: `Invalid image attachment payload for '${attachment.name}'.`,
116+
message: `Invalid ${attachment.type} attachment payload for '${attachment.name}'.`,
115117
});
116118
}
117119

118120
const bytes = Buffer.from(parsed.base64, "base64");
119-
if (bytes.byteLength === 0 || bytes.byteLength > PROVIDER_SEND_TURN_MAX_IMAGE_BYTES) {
121+
const maxBytes =
122+
attachment.type === "audio"
123+
? PROVIDER_SEND_TURN_MAX_AUDIO_BYTES
124+
: PROVIDER_SEND_TURN_MAX_IMAGE_BYTES;
125+
if (bytes.byteLength === 0 || bytes.byteLength > maxBytes) {
120126
return yield* new OrchestrationDispatchCommandError({
121-
message: `Image attachment '${attachment.name}' is empty or too large.`,
127+
message: `${attachment.type === "audio" ? "Audio" : "Image"} attachment '${attachment.name}' is empty or too large.`,
122128
});
123129
}
124130

@@ -129,13 +135,25 @@ export const normalizeDispatchCommand = (command: ClientOrchestrationCommand) =>
129135
});
130136
}
131137

132-
const persistedAttachment = {
133-
type: "image" as const,
134-
id: attachmentId,
135-
name: attachment.name,
136-
mimeType: parsed.mimeType.toLowerCase(),
137-
sizeBytes: bytes.byteLength,
138-
};
138+
const persistedAttachment =
139+
attachment.type === "audio"
140+
? {
141+
type: "audio" as const,
142+
id: attachmentId,
143+
name: attachment.name,
144+
mimeType: parsed.mimeType.toLowerCase(),
145+
sizeBytes: bytes.byteLength,
146+
durationMs: attachment.durationMs,
147+
waveform: attachment.waveform,
148+
transcriptionStatus: "transcribing" as const,
149+
}
150+
: {
151+
type: "image" as const,
152+
id: attachmentId,
153+
name: attachment.name,
154+
mimeType: parsed.mimeType.toLowerCase(),
155+
sizeBytes: bytes.byteLength,
156+
};
139157

140158
const attachmentPath = resolveAttachmentPath({
141159
attachmentsDir: serverConfig.attachmentsDir,

‎apps/server/src/provider/Drivers/HermesDriver.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,13 @@ export function makeHermesProviderSnapshot(input: {
144144
badgeLabel: "Agent",
145145
showInteractionModeToggle: false,
146146
requiresNewThreadForModelChange: false,
147+
...(input.capabilities?.capabilities.voiceNotes
148+
? {
149+
voiceNotes: {
150+
maxBytes: input.capabilities.capabilities.voiceNoteMaxBytes ?? 128 * 1024 * 1024,
151+
},
152+
}
153+
: {}),
147154
enabled: input.enabled,
148155
installed: true,
149156
version: null,

‎apps/server/src/provider/Layers/HermesAdapter.ts‎

Lines changed: 86 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import {
22
ApprovalRequestId,
33
EventId,
4+
HermesBridgeAudioAttachmentId,
45
type HermesBridgeApprovalRequest,
56
HermesBridgeChatId,
67
type HermesBridgeChoice,
@@ -724,6 +725,63 @@ export const makeHermesAdapter = Effect.fn("makeHermesAdapter")(function* (
724725
};
725726
break;
726727
}
728+
case "token-usage.updated": {
729+
const base = yield* eventBase(callback, threadId, "token-usage");
730+
yield* publish({
731+
...base,
732+
type: "thread.token-usage.updated",
733+
payload: {
734+
usage: {
735+
usedTokens: callback.usedTokens,
736+
maxTokens: callback.maxTokens,
737+
...(callback.totalProcessedTokens !== undefined
738+
? { totalProcessedTokens: callback.totalProcessedTokens }
739+
: {}),
740+
...(callback.inputTokens !== undefined
741+
? { inputTokens: callback.inputTokens }
742+
: {}),
743+
...(callback.cachedInputTokens !== undefined
744+
? { cachedInputTokens: callback.cachedInputTokens }
745+
: {}),
746+
...(callback.outputTokens !== undefined
747+
? { outputTokens: callback.outputTokens }
748+
: {}),
749+
...(callback.reasoningOutputTokens !== undefined
750+
? { reasoningOutputTokens: callback.reasoningOutputTokens }
751+
: {}),
752+
compactsAutomatically: callback.compactsAutomatically,
753+
},
754+
},
755+
});
756+
break;
757+
}
758+
case "voice.transcription": {
759+
const base = yield* eventBase(callback, threadId, "voice-transcription");
760+
yield* publish({
761+
...base,
762+
type: "item.updated",
763+
itemId: RuntimeItemId.make(`hermes-voice:${callback.messageId}`),
764+
payload: {
765+
itemType: "user_message",
766+
status: callback.status === "failed" ? "failed" : "completed",
767+
title:
768+
callback.status === "transcribing"
769+
? "Transcribing voice note"
770+
: callback.status === "failed"
771+
? "Voice transcription failed"
772+
: "Voice note transcribed",
773+
data: {
774+
messageId: callback.messageId,
775+
status: callback.status,
776+
...(callback.transcript !== undefined
777+
? { transcript: callback.transcript }
778+
: {}),
779+
...(callback.error !== undefined ? { error: callback.error } : {}),
780+
},
781+
},
782+
});
783+
break;
784+
}
727785
case "session.title.updated": {
728786
const base = yield* eventBase(callback, threadId, "title");
729787
yield* publish({
@@ -856,7 +914,7 @@ export const makeHermesAdapter = Effect.fn("makeHermesAdapter")(function* (
856914
activeTurnId: turnId,
857915
updatedAt: createdAt,
858916
};
859-
const images = yield* Effect.forEach(input.attachments ?? [], (attachment) => {
917+
const bridgeAttachments = yield* Effect.forEach(input.attachments ?? [], (attachment) => {
860918
const attachmentPath = resolveAttachmentPath({
861919
attachmentsDir: serverConfig.attachmentsDir,
862920
attachment,
@@ -870,15 +928,33 @@ export const makeHermesAdapter = Effect.fn("makeHermesAdapter")(function* (
870928
}),
871929
);
872930
}
873-
return Effect.succeed({
874-
type: "image" as const,
875-
id: HermesBridgeImageAttachmentId.make(attachment.id),
876-
name: attachment.name,
877-
mimeType: attachment.mimeType,
878-
sizeBytes: attachment.sizeBytes,
879-
source: { type: "local-path" as const, path: attachmentPath },
880-
});
931+
return Effect.succeed(
932+
attachment.type === "image"
933+
? {
934+
type: "image" as const,
935+
id: HermesBridgeImageAttachmentId.make(attachment.id),
936+
name: attachment.name,
937+
mimeType: attachment.mimeType,
938+
sizeBytes: attachment.sizeBytes,
939+
source: { type: "local-path" as const, path: attachmentPath },
940+
}
941+
: {
942+
type: "audio" as const,
943+
id: HermesBridgeAudioAttachmentId.make(attachment.id),
944+
name: attachment.name,
945+
mimeType: attachment.mimeType,
946+
sizeBytes: attachment.sizeBytes,
947+
durationMs: attachment.durationMs,
948+
waveform: attachment.waveform,
949+
source: { type: "local-path" as const, path: attachmentPath },
950+
},
951+
);
881952
});
953+
const images = bridgeAttachments.filter(
954+
(attachment): attachment is Extract<(typeof bridgeAttachments)[number], { type: "image" }> =>
955+
attachment.type === "image",
956+
);
957+
const audio = bridgeAttachments.find((attachment) => attachment.type === "audio");
882958
yield* publish({
883959
eventId: EventId.make(`hermes:${turnId}:started`),
884960
provider: PROVIDER,
@@ -916,6 +992,7 @@ export const makeHermesAdapter = Effect.fn("makeHermesAdapter")(function* (
916992
}
917993
: {}),
918994
...(images.length > 0 ? { images } : {}),
995+
...(audio ? { audio } : {}),
919996
})
920997
.pipe(
921998
Effect.flatMap((acknowledgement) =>

0 commit comments

Comments
 (0)