diff --git a/include/parakeet_capi.h b/include/parakeet_capi.h index e84e485..d4a5565 100644 --- a/include/parakeet_capi.h +++ b/include/parakeet_capi.h @@ -40,6 +40,13 @@ typedef struct parakeet_ctx parakeet_ctx; // Added parakeet_capi_stream_drain_events (typed per-event records with // is_eob + timestamps, freed with parakeet_capi_free_events) and an // "events" array in the stream_feed_json / stream_finalize_json documents. +// +// v6: added parakeet_capi_transcribe_pcm_logits, exposing the CTC head's +// log-prob matrix (row-major [T, vocab+1], already log-softmaxed) instead +// of decoded text — for external LM/decoder stacks (e.g. pyctcdecode + +// KenLM) that need the raw distribution rather than this library's own +// greedy/beam decode. Freed with the new parakeet_capi_free_logits. The +// original entry points are unchanged. int parakeet_capi_abi_version(void); // Load a GGUF model. Returns an owning context, or NULL on failure. @@ -180,6 +187,35 @@ char* parakeet_capi_transcribe_pcm_nbest_json( parakeet_ctx* ctx, const float* samples, int n_samples, int sample_rate, int beam_size, int nbest, int score_norm, const char* target_lang); +// Run mel + encoder + CTC head on in-memory mono float PCM and return the +// log-prob matrix instead of decoded text, for callers that run their own +// external decoder (e.g. pyctcdecode + a KenLM n-gram LM + hotwords) on top of +// this library's CTC output rather than using parakeet.cpp's own greedy/beam +// decode. If `sample_rate != 16000` the audio is linearly resampled to 16 kHz +// first. Always runs the CTC head regardless of the model's preferred +// decoder — `decoder` is not a parameter here, unlike parakeet_capi_transcribe_pcm. +// +// On success returns 0, mallocs `*out_logits` to `(*out_T) * (*out_vocab_plus_1)` +// floats — row-major [T, vocab+1], i.e. out_logits[t*(*out_vocab_plus_1) + v], +// already log-softmaxed over the vocab axis — and sets `*out_T` / +// `*out_vocab_plus_1`. Free `*out_logits` with parakeet_capi_free_logits. +// +// On error returns nonzero. A NULL `ctx` or any NULL out-param pointer +// returns nonzero without writing through any pointer (nothing to zero +// safely). Otherwise (ctx and all three out-params valid, but e.g. no model, +// invalid samples buffer, the model has no CTC head, or OOM) sets the +// context's last error (see parakeet_capi_last_error) and leaves `*out_logits` +// NULL and `*out_T`/`*out_vocab_plus_1` 0 — the caller owns nothing and has +// nothing to free. +int parakeet_capi_transcribe_pcm_logits(parakeet_ctx* ctx, const float* samples, + int n_samples, int sample_rate, + float** out_logits, int* out_T, + int* out_vocab_plus_1); + +// Free a logits buffer previously returned by +// parakeet_capi_transcribe_pcm_logits. Safe on NULL. +void parakeet_capi_free_logits(float* logits); + // --------------------------------------------------------------------------- // Streaming API (cache-aware streaming RNN-T, e.g. the EOU model // nvidia/parakeet_realtime_eou_120m-v1). The stream session buffers incoming diff --git a/src/model.cpp b/src/model.cpp index e3ae0c1..b9d812a 100644 --- a/src/model.cpp +++ b/src/model.cpp @@ -183,6 +183,57 @@ std::string Model::transcribe_16k(const std::vector& pcm16k, encoded.d_model, encoded.frames, use_tdt); } +void Model::transcribe_16k_ctc_logits(const std::vector& pcm16k, + std::vector& logits, int& T, + int& vocab_plus_1, + const std::string& target_lang) const { + const ParakeetConfig& cfg = loader_.config(); + const int prompt_index = resolve_prompt_index(target_lang); + + // 1. Log-mel front end -> feats [n_mels, T]. Mirrors transcribe_16k exactly. + std::vector feats; + int n_mels = 0, Tmel = 0; + if (std::string(pk::global_backend().device_name()) != "cpu") { + GpuMel gmel(loader_); + gmel.compute(pcm16k, feats, n_mels, Tmel); + } else { + MelFrontend mel(loader_); + mel.compute(pcm16k, feats, n_mels, Tmel); + } + + // 2. FastConformer encoder -> enc_out [d_model, Tout] (channels-first). + // Long audio: tile the subsampling stage exactly as transcribe_16k does. + Encoder encoder(loader_); + std::vector enc_out; + int d_model = 0, Tout = 0; + const int sub_tile = subsampling_tile_for(cfg, loader_, Tmel); + if (sub_tile > 0) { + MelBatch mb1; + mb1.B = 1; mb1.n_mels = n_mels; mb1.T_max = Tmel; mb1.valid_T = { Tmel }; + mb1.data = feats; + std::vector> eo; std::vector vT; + int dm1 = 0, To1 = 0; + encoder.forward_batch_tiled(mb1, eo, dm1, To1, vT, sub_tile); + enc_out = std::move(eo[0]); + d_model = dm1; + Tout = vT[0]; + } else { + encoder.forward(feats, n_mels, Tmel, enc_out, d_model, Tout); + } + + // 2b. Prompt conditioning (multilingual nemotron): project the encoder + // output with the selected language one-hot before decoding. No-op + // for other models (prompt.present == false). + maybe_apply_prompt(loader_, enc_out, d_model, Tout, prompt_index); + + // 3. CTC head only — always, regardless of the model's preferred decoder. + // Throws std::runtime_error (from ctc_head_tensor, via CTCDecoder::forward) + // if the model has no CTC head, e.g. a TDT/RNNT-only streaming model. + CTCDecoder ctc(loader_); + ctc.forward(enc_out, d_model, Tout, logits, vocab_plus_1); + T = Tout; +} + // Max mel frames per encoder pass before the first subsampling conv output // (n_mels/2 * T/2 * conv_channels) approaches INT_MAX. ggml's CUDA unary (relu) // kernel indexes elements with int32, so a tensor > 2^31 elements crashes @@ -555,6 +606,21 @@ std::string Model::transcribe_pcm(const std::vector& pcm, int sample_rate return transcribe_16k(pcm16k, decoder, target_lang); } +void Model::transcribe_pcm_ctc_logits(const std::vector& pcm, int sample_rate, + std::vector& logits, int& T, + int& vocab_plus_1, + const std::string& target_lang) const { + if (sample_rate <= 0) { + throw std::runtime_error("parakeet: invalid sample_rate"); + } + if (sample_rate == 16000) { + transcribe_16k_ctc_logits(pcm, logits, T, vocab_plus_1, target_lang); + return; + } + std::vector pcm16k = resample_linear(pcm, sample_rate, 16000); + transcribe_16k_ctc_logits(pcm16k, logits, T, vocab_plus_1, target_lang); +} + std::string Model::transcribe_path(const std::string& wav_path, Decoder decoder, const std::string& target_lang) const { Audio audio; diff --git a/src/model.hpp b/src/model.hpp index 4ff6fa4..e3d280e 100644 --- a/src/model.hpp +++ b/src/model.hpp @@ -70,6 +70,19 @@ class Model { Decoder decoder = Decoder::kDefault, const std::string& target_lang = "") const; + // Run mel + encoder + CTC head only, returning the log-prob matrix + // (row-major [T, vocab+1], already log-softmaxed) instead of decoded text — + // the seam external decoder stacks (e.g. pyctcdecode + KenLM) need. If + // `sample_rate != 16000` the audio is linearly resampled to 16 kHz first. + // `target_lang` as in transcribe_pcm (ignored by non-prompt models). Always + // runs the CTC head regardless of the model's preferred decoder; throws + // std::runtime_error if the model has no CTC head (e.g. a TDT/RNNT-only + // streaming model). + void transcribe_pcm_ctc_logits(const std::vector& pcm, int sample_rate, + std::vector& logits, int& T, + int& vocab_plus_1, + const std::string& target_lang = "") const; + // Transcribe raw mono float PCM, returning the flat text plus per-word and // per-token timestamps + confidence (matching NeMo timestamps=True + // 'max_prob' confidence). If `sample_rate != 16000` the audio is linearly @@ -141,6 +154,14 @@ class Model { const std::vector& pcm16k, int beam_size, int nbest, bool score_norm, const std::string& target_lang) const; + // Core orchestration for transcribe_pcm_ctc_logits: 16 kHz mono PCM -> CTC + // log-prob matrix. Mirrors transcribe_16k through the encoder, then runs + // the CTC head directly instead of decode_enc_out. + void transcribe_16k_ctc_logits(const std::vector& pcm16k, + std::vector& logits, int& T, + int& vocab_plus_1, + const std::string& target_lang = "") const; + ModelLoader loader_; }; diff --git a/src/parakeet_capi.cpp b/src/parakeet_capi.cpp index cfe481b..1a6c9b5 100644 --- a/src/parakeet_capi.cpp +++ b/src/parakeet_capi.cpp @@ -32,7 +32,10 @@ // Added stream_drain_events / free_events (typed per-event records) and // the "events" array in the stream_feed_json / stream_finalize_json // documents. -#define PARAKEET_CAPI_ABI_VERSION 5 +// v6: transcribe_pcm_logits, exposing the CTC head's log-prob matrix (row-major +// [T, vocab+1], already log-softmaxed) instead of decoded text, freed with +// the new free_logits. Original entry points unchanged. +#define PARAKEET_CAPI_ABI_VERSION 6 // The opaque context: a loaded model plus a buffer for the last error message. struct parakeet_ctx { @@ -205,6 +208,48 @@ extern "C" char* parakeet_capi_transcribe_pcm(parakeet_ctx* ctx, const float* sa decoder, nullptr); } +extern "C" int parakeet_capi_transcribe_pcm_logits(parakeet_ctx* ctx, + const float* samples, int n_samples, + int sample_rate, float** out_logits, + int* out_T, int* out_vocab_plus_1) { + if (!ctx) return 1; + if (!out_logits || !out_T || !out_vocab_plus_1) { + ctx->last_error = "invalid output pointer(s)"; + return 1; + } + *out_logits = nullptr; + *out_T = 0; + *out_vocab_plus_1 = 0; + if (!ctx->model) { ctx->last_error = "context has no loaded model"; return 1; } + if (!samples || n_samples < 0) { ctx->last_error = "invalid samples buffer"; return 1; } + try { + std::vector pcm(samples, samples + n_samples); + std::vector logits; + int T = 0, vocab_plus_1 = 0; + ctx->model->transcribe_pcm_ctc_logits(pcm, sample_rate, logits, T, vocab_plus_1); + + float* buf = static_cast(std::malloc(logits.size() * sizeof(float))); + if (!buf) { ctx->last_error = "out of memory"; return 1; } + std::memcpy(buf, logits.data(), logits.size() * sizeof(float)); + + ctx->last_error.clear(); + *out_logits = buf; + *out_T = T; + *out_vocab_plus_1 = vocab_plus_1; + return 0; + } catch (const std::exception& e) { + ctx->last_error = e.what(); + return 1; + } catch (...) { + ctx->last_error = "unknown error"; + return 1; + } +} + +extern "C" void parakeet_capi_free_logits(float* logits) { + std::free(logits); +} + extern "C" int parakeet_capi_transcribe_pcm_batch_lang(parakeet_ctx* ctx, const float* const* samples, const int* n_samples, int n_clips, diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 126b48e..d828c6e 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -69,6 +69,7 @@ pk_add_test(test_capi_stream) pk_add_test(test_capi_stream_json) pk_add_test(test_capi_timestamps) pk_add_test(test_capi_batch_json) +pk_add_test(test_capi_ctc_logits) if(TARGET parakeet-cli) add_test(NAME cli_version_long COMMAND $ --version) @@ -125,7 +126,7 @@ set_tests_properties(test_model_loader test_mel test_mel_gpu test_subsampling te test_transcribe_speech test_transcribe_tiled test_transcribe_tdt test_transcribe_0_6b test_transcribe_ctc test_transcribe_rnnt test_transcribe_eou test_transcribe_nemotron test_streaming_decode test_streaming_eou_reset test_streaming_nemotron test_streaming_mel test_capi test_capi_batch test_capi_stream test_capi_stream_json - test_capi_timestamps test_capi_batch_json + test_capi_timestamps test_capi_batch_json test_capi_ctc_logits PROPERTIES LABELS "model") # These tests read fixtures/baselines via paths relative to the project root. set_tests_properties(test_mel test_mel_gpu test_subsampling test_subsampling_batch test_subsampling_batch_causal test_relpos_attention test_relpos_attention_batch test_conformer test_conformer_batch @@ -141,7 +142,7 @@ set_tests_properties(test_mel test_mel_gpu test_subsampling test_subsampling_bat test_transcribe_speech test_transcribe_tiled test_transcribe_tdt test_transcribe_0_6b test_transcribe_ctc test_transcribe_rnnt test_transcribe_eou test_transcribe_nemotron test_streaming_decode test_streaming_eou_reset test_streaming_nemotron test_streaming_mel test_capi test_capi_batch test_capi_stream test_capi_stream_json - test_capi_timestamps test_capi_batch_json + test_capi_timestamps test_capi_batch_json test_capi_ctc_logits PROPERTIES WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) # Python converter check (skips with exit 77 when the venv/model are absent). diff --git a/tests/test_capi_ctc_logits.cpp b/tests/test_capi_ctc_logits.cpp new file mode 100644 index 0000000..719bb67 --- /dev/null +++ b/tests/test_capi_ctc_logits.cpp @@ -0,0 +1,190 @@ +#include "model.hpp" +#include "audio_io.hpp" +#include "search.hpp" +#include "tokenizer.hpp" +#include "parakeet_capi.h" +#include +#include +#include +#include +#include + +// Coverage for parakeet_capi_transcribe_pcm_logits (the classroom-captions#63 +// logits-exposure entry point), in two independent blocks (mirrors +// test_capi.cpp's two-optional-env-vars shape): +// +// 1. Self-consistency on a real standalone-CTC checkpoint, through the +// actual C-API (parakeet_capi_load / parakeet_capi_transcribe_pcm_logits +// / parakeet_capi_free_logits — exercising the C boundary and malloc/free +// contract, not just the underlying C++ method): reconstructing text +// from the exposed [T, vocab+1] log-prob matrix via the SAME ctc_greedy + +// detokenize path decode_enc_out uses internally must reproduce +// transcribe_pcm(..., kCTC)'s own greedy transcript byte-for-byte +// (transcribe_pcm and the tokenizer/blank_id come from a separate +// pk::Model load, used only for that reference text and metadata). +// Exercises the ctc_head_tensor standalone-model fallback path +// (decoder.* prefix, not the hybrid ctc_decoder.*). +// +// 2. Error path at the C-API boundary: a model with NO CTC head at all +// (e.g. a pure RNNT/TDT streaming model) must make +// parakeet_capi_transcribe_pcm_logits fail cleanly — nonzero return, +// *out_logits left NULL, ctx last_error set — never crash or let the +// underlying std::runtime_error (from ctc_head_tensor) cross the C +// boundary. +// +// This is a self-consistency test (our own greedy decode vs. our own exposed +// logits, both computed here), not a NeMo parity check — that's already +// covered by test_transcribe_ctc.cpp / test_ctc.cpp. +// +// Env: +// PARAKEET_TEST_GGUF_CTC standalone CTC GGUF (block 1; skip if unset) +// PARAKEET_TEST_GGUF_NO_CTC a GGUF with no CTC head, e.g. a pure RNNT/TDT +// streaming model (block 2; skip if unset) +// +// LABEL model +// WORKING_DIRECTORY (tests run from the project root; wav path is relative) +int main() { + bool ran_any = false; + + const char* ctc_gguf = std::getenv("PARAKEET_TEST_GGUF_CTC"); + if (ctc_gguf) { + ran_any = true; + // pk::Model is used only for the reference text and tokenizer/blank_id + // access below — the logits themselves come from the actual C-API + // (parakeet_capi_load/parakeet_capi_transcribe_pcm_logits), so this + // block exercises the C boundary and malloc/free contract, not just + // the underlying C++ method. + auto model = pk::Model::load(ctc_gguf); + if (!model) { + std::fprintf(stderr, "test_capi_ctc_logits: load failed for %s\n", ctc_gguf); + return 1; + } + + pk::Audio audio; + if (!pk::load_audio_16k_mono("tests/fixtures/speech.wav", audio) || audio.samples.empty()) { + std::fprintf(stderr, "test_capi_ctc_logits: wav load failed\n"); + return 1; + } + + const std::string reference = model->transcribe_pcm(audio.samples, 16000, pk::Decoder::kCTC); + const int blank_id = (int)model->config().blank_id; + const std::vector tokenizer_pieces = model->loader().tokenizer_pieces(); + // Release the pk::Model before loading a second full model via the C-API + // below — keeping both resident at once nearly doubles peak RAM for a + // 1.1B checkpoint. Only blank_id/tokenizer_pieces (cached above) and the + // already-computed reference text are needed from here on. + model.reset(); + + parakeet_ctx* ctx = parakeet_capi_load(ctc_gguf); + if (!ctx) { + std::fprintf(stderr, "test_capi_ctc_logits: parakeet_capi_load failed for %s\n", ctc_gguf); + return 1; + } + + float* out_logits = nullptr; + int T = 0, vocab_plus_1 = 0; + int rc = parakeet_capi_transcribe_pcm_logits( + ctx, audio.samples.data(), (int)audio.samples.size(), 16000, + &out_logits, &T, &vocab_plus_1); + + if (rc != 0) { + std::fprintf(stderr, "test_capi_ctc_logits: transcribe_pcm_logits failed: %s\n", + parakeet_capi_last_error(ctx)); + parakeet_capi_free(ctx); + return 1; + } + if (!out_logits || T <= 0 || vocab_plus_1 <= 0) { + std::fprintf(stderr, + "test_capi_ctc_logits: bad output out_logits=%p T=%d vocab_plus_1=%d\n", + (void*)out_logits, T, vocab_plus_1); + parakeet_capi_free_logits(out_logits); + parakeet_capi_free(ctx); + return 1; + } + + std::vector logits(out_logits, out_logits + (size_t)T * (size_t)vocab_plus_1); + parakeet_capi_free_logits(out_logits); + parakeet_capi_free(ctx); + + std::vector ids = pk::ctc_greedy(logits, T, vocab_plus_1, blank_id); + const std::string reconstructed = pk::detokenize( + tokenizer_pieces, + pk::strip_special_tokens(tokenizer_pieces, ids)); + + std::fprintf(stderr, "test_capi_ctc_logits: reference = %s\n", reference.c_str()); + std::fprintf(stderr, "test_capi_ctc_logits: reconstructed = %s\n", reconstructed.c_str()); + std::fprintf(stderr, "test_capi_ctc_logits: T=%d vocab_plus_1=%d blank_id=%d\n", + T, vocab_plus_1, blank_id); + + if (reconstructed != reference) { + std::fprintf(stderr, "test_capi_ctc_logits: MISMATCH\n"); + return 1; + } + + std::fprintf(stderr, + "test_capi_ctc_logits: PASS block 1 (argmax-greedy over the C-API's exposed " + "logits reproduces the CLI's own greedy text)\n"); + } else { + std::fprintf(stderr, "test_capi_ctc_logits: PARAKEET_TEST_GGUF_CTC not set; skip block 1\n"); + } + + const char* no_ctc_gguf = std::getenv("PARAKEET_TEST_GGUF_NO_CTC"); + if (no_ctc_gguf) { + ran_any = true; + parakeet_ctx* ctx = parakeet_capi_load(no_ctc_gguf); + if (!ctx) { + std::fprintf(stderr, "test_capi_ctc_logits: load failed for %s\n", no_ctc_gguf); + return 1; + } + + pk::Audio audio; + if (!pk::load_audio_16k_mono("tests/fixtures/speech.wav", audio) || audio.samples.empty()) { + std::fprintf(stderr, "test_capi_ctc_logits: wav load failed\n"); + parakeet_capi_free(ctx); + return 1; + } + + float* out_logits = nullptr; + int out_T = 0, out_vocab_plus_1 = 0; + int rc = parakeet_capi_transcribe_pcm_logits( + ctx, audio.samples.data(), (int)audio.samples.size(), 16000, + &out_logits, &out_T, &out_vocab_plus_1); + + if (rc == 0) { + std::fprintf(stderr, + "test_capi_ctc_logits: expected failure on a no-CTC-head model, got rc=0\n"); + parakeet_capi_free_logits(out_logits); + parakeet_capi_free(ctx); + return 1; + } + if (out_logits != nullptr) { + std::fprintf(stderr, + "test_capi_ctc_logits: rc!=0 but *out_logits is non-NULL (ownership contract violated)\n"); + parakeet_capi_free_logits(out_logits); + parakeet_capi_free(ctx); + return 1; + } + const char* err = parakeet_capi_last_error(ctx); + if (!err || err[0] == '\0') { + std::fprintf(stderr, "test_capi_ctc_logits: no-CTC-head failure did not set last_error\n"); + parakeet_capi_free(ctx); + return 1; + } + std::fprintf(stderr, "test_capi_ctc_logits: no-CTC-head error (expected) = %s\n", err); + + parakeet_capi_free(ctx); + std::fprintf(stderr, + "test_capi_ctc_logits: PASS block 2 (no-CTC-head model fails cleanly, " + "no crash, last_error set)\n"); + } else { + std::fprintf(stderr, "test_capi_ctc_logits: PARAKEET_TEST_GGUF_NO_CTC not set; skip block 2\n"); + } + + if (!ran_any) { + std::fprintf(stderr, + "test_capi_ctc_logits: no model env var set (PARAKEET_TEST_GGUF_CTC / " + "PARAKEET_TEST_GGUF_NO_CTC); skip\n"); + return 77; + } + return 0; +}