diff --git a/cookbooks/demo.ipynb b/cookbooks/demo.ipynb index 477b54a..f9f8fb5 100644 --- a/cookbooks/demo.ipynb +++ b/cookbooks/demo.ipynb @@ -1,831 +1 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "4770bf52-16aa-486e-9a5e-dad64a8ad658", - "metadata": { - "execution": { - "shell.execute_reply.end": "2025-09-30T12:53:17.283519Z", - "shell.execute_reply.started": "2025-09-30T12:53:17.280894Z", - "to_execute": "2025-09-30T12:53:17.215Z" - }, - "isLargeOutputDisplay": true, - "libroFormatter": "formatter-string", - "trusted": true - }, - "outputs": [], - "source": [ - "%cd /path/to/workspace" - ] - }, - { - "cell_type": "markdown", - "id": "a28108cc-d941-4910-ac00-6106e68c2d99", - "metadata": { - "libroFormatter": "formatter-string", - "trusted": true - }, - "source": [ - "# Initialize" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e00bfc4a-38b2-4afa-899b-1ea0c6483c7b", - "metadata": { - "execution": { - "shell.execute_reply.end": "2025-09-30T12:54:16.424007Z", - "shell.execute_reply.started": "2025-09-30T12:53:47.362548Z", - "to_execute": "2025-09-30T12:53:47.307Z" - }, - "isLargeOutputDisplay": true, - "libroFormatter": "formatter-string", - "trusted": true - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/opt/conda/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n", - "[2025-09-30 20:53:52,335] [INFO] [add_hooks.py:71:__init__] If you are working on AIStudio, please set 'ANTMONITOR_TFEVENT_PATH' env.\n", - "Loading checkpoint shards: 100%|██████████| 8/8 [00:02<00:00, 3.85it/s]\n", - "Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\n" - ] - } - ], - "source": [ - "import warnings\n", - "import torch\n", - "from transformers import AutoProcessor\n", - "\n", - "from modeling_bailingmm import BailingMMNativeForConditionalGeneration\n", - "\n", - "import random\n", - "import numpy as np\n", - "from loguru import logger\n", - "\n", - "def seed_everything(seed=1895):\n", - " random.seed(seed)\n", - " np.random.seed(seed)\n", - " torch.manual_seed(seed)\n", - " torch.cuda.manual_seed(seed)\n", - " torch.cuda.manual_seed_all(seed)\n", - " torch.backends.cudnn.deterministic = True\n", - " torch.backends.cudnn.benchmark = False\n", - "\n", - "seed_everything()\n", - "warnings.filterwarnings(\"ignore\")\n", - "\n", - "class MingAudio:\n", - " def __init__(self, model_path, device=\"cuda:0\"):\n", - " self.device = device\n", - " self.model = BailingMMNativeForConditionalGeneration.from_pretrained(\n", - " model_path,\n", - " torch_dtype=torch.bfloat16,\n", - " low_cpu_mem_usage=True,\n", - " ).eval().to(torch.bfloat16).to(self.device)\n", - " self.processor = AutoProcessor.from_pretrained(\".\", trust_remote_code=True)\n", - " self.tokenizer = self.processor.tokenizer\n", - " self.sample_rate = self.processor.audio_processor.sample_rate\n", - " self.patch_size = self.processor.audio_processor.patch_size\n", - " \n", - " def speech_understanding(self, messages, lang=None):\n", - " text = self.processor.apply_chat_template(messages, add_generation_prompt=True)\n", - " image_inputs, video_inputs, audio_inputs = self.processor.process_vision_info(messages)\n", - "\n", - " inputs = self.processor(\n", - " text=[text],\n", - " images=image_inputs,\n", - " videos=video_inputs,\n", - " audios=audio_inputs,\n", - " return_tensors=\"pt\",\n", - " ).to(self.device)\n", - " \n", - " if lang is not None:\n", - " language = torch.tensor([self.tokenizer.encode(f'{lang}\\t')]).to(inputs['input_ids'].device)\n", - " inputs['input_ids'] = torch.cat([inputs['input_ids'], language], dim=1)\n", - " attention_mask = inputs['attention_mask']\n", - " inputs['attention_mask'] = torch.ones(inputs['input_ids'].shape, dtype=attention_mask.dtype)\n", - " for k in inputs.keys():\n", - " if k == \"pixel_values\" or k == \"pixel_values_videos\" or k == \"audio_feats\":\n", - " inputs[k] = inputs[k].to(dtype=torch.bfloat16)\n", - " logger.info(f\"input: {self.tokenizer.decode(inputs['input_ids'].cpu().numpy().tolist()[0])}\")\n", - "\n", - " generated_ids = self.model.generate(\n", - " **inputs,\n", - " max_new_tokens=512,\n", - " eos_token_id=self.processor.gen_terminator,\n", - " )\n", - " generated_ids_trimmed = [\n", - " out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)\n", - " ]\n", - " output_text = self.processor.batch_decode(\n", - " generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False\n", - " )[0]\n", - "\n", - " return output_text\n", - "\n", - " def speech_generation(\n", - " self, \n", - " text,\n", - " prompt_wav_path,\n", - " prompt_text,\n", - " lang='zh',\n", - " output_wav_path='out.wav'\n", - " ):\n", - " waveform = self.model.generate_tts(\n", - " text=text,\n", - " prompt_wav_path=prompt_wav_path,\n", - " prompt_text=prompt_text,\n", - " patch_size=self.patch_size,\n", - " tokenizer=self.tokenizer,\n", - " lang=lang,\n", - " output_wav_path=output_wav_path,\n", - " sample_rate=self.sample_rate,\n", - " device=self.device\n", - " )\n", - " \n", - " return waveform\n", - "\n", - " def speech_edit(\n", - " self, \n", - " messages,\n", - " output_wav_path='out.wav',\n", - " use_cot=True\n", - " ):\n", - " text = self.processor.apply_chat_template(messages, add_generation_prompt=True)\n", - " image_inputs, video_inputs, audio_inputs = self.processor.process_vision_info(messages)\n", - "\n", - " inputs = self.processor(\n", - " text=[text],\n", - " images=image_inputs,\n", - " videos=video_inputs,\n", - " audios=audio_inputs,\n", - " return_tensors=\"pt\",\n", - " ).to(self.device)\n", - "\n", - " if use_cot:\n", - " ans = torch.tensor([self.tokenizer.encode('')]).to(inputs['input_ids'].device)\n", - " inputs['input_ids'] = torch.cat([inputs['input_ids'], ans], dim=1)\n", - " attention_mask = inputs['attention_mask']\n", - " inputs['attention_mask'] = torch.ones(inputs['input_ids'].shape, dtype=attention_mask.dtype)\n", - " for k in inputs.keys():\n", - " if k == \"pixel_values\" or k == \"pixel_values_videos\" or k == \"audio_feats\":\n", - " inputs[k] = inputs[k].to(dtype=torch.bfloat16)\n", - " logger.info(f\"input: {self.tokenizer.decode(inputs['input_ids'].cpu().numpy().tolist()[0])}\")\n", - "\n", - " edited_speech, edited_text = self.model.generate_edit(\n", - " **inputs,\n", - " tokenizer=self.tokenizer,\n", - " output_wav_path=output_wav_path\n", - " )\n", - " return edited_speech, edited_text\n", - "\n", - "\n", - "if __name__ == \"__main__\":\n", - " # model = MingAudio(\"/path/to/model\")\n", - " # load base model\n", - " model = MingAudio(\"inclusionAI/Ming-UniAudio-16B-A3B\")\n" - ] - }, - { - "cell_type": "markdown", - "id": "ff8c3e1e-690a-4232-94b6-3d0ce26879fa", - "metadata": { - "libroFormatter": "formatter-string", - "trusted": true - }, - "source": [ - "# Examples" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "170c71f9-336a-433a-9e4a-a6980b938e4d", - "metadata": { - "execution": { - "shell.execute_reply.end": "2025-09-30T12:54:28.288569Z", - "shell.execute_reply.started": "2025-09-30T12:54:26.623333Z", - "to_execute": "2025-09-30T12:54:26.556Z" - }, - "isLargeOutputDisplay": true, - "libroFormatter": "formatter-string", - "trusted": true - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[32m2025-09-30 20:54:26.673\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36mspeech_understanding\u001b[0m:\u001b[36m56\u001b[0m - \u001b[1minput: HUMANPlease recognize the language of this speech and transcribe it. Format: oral.ASSISTANT\u001b[0m\n", - "The `seen_tokens` attribute is deprecated and will be removed in v4.41. Use the `cache_position` model input instead.\n", - "2025-09-30 20:54:27,589 - modeling_bailing_moe - WARNING - The input hidden states seems to be silently casted in float32, this might be related to the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in torch.bfloat16.\n", - "\u001b[32m2025-09-30 20:54:28.285\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m\u001b[0m:\u001b[36m17\u001b[0m - \u001b[1mGenerated Response: Chinese\t现在是不是也该长点心了吧\u001b[0m\n" - ] - } - ], - "source": [ - "\n", - " # ASR\n", - " messages = [\n", - " {\n", - " \"role\": \"HUMAN\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"text\",\n", - " \"text\": \"Please recognize the language of this speech and transcribe it. Format: oral.\",\n", - " },\n", - " \n", - " {\"type\": \"audio\", \"audio\": \"data/wavs/BAC009S0915W0292.wav\"},\n", - " ],\n", - " },\n", - " ]\n", - " \n", - " response = model.speech_understanding(messages=messages)\n", - " logger.info(f\"Generated Response: {response}\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "41cc7088-a1ec-4e17-84d3-6047a30a6557", - "metadata": { - "execution": { - "shell.execute_reply.end": "2025-09-30T12:56:01.997703Z", - "shell.execute_reply.started": "2025-09-30T12:55:48.795689Z", - "to_execute": "2025-09-30T12:55:48.789Z" - }, - "isLargeOutputDisplay": true, - "libroFormatter": "formatter-string", - "trusted": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "evaluating zh\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " 0%| | 0/300 [00:00HUMANPlease translate the text to speech.\n", - "在此奉劝大家别乱打美白针。我们的愿景是构建未来服务业的数字化基础设施，为世界带来更多微小而美好的改变。ASSISTANT