A Hands-On Coding Tutorial for Microsoft VibeVoice Covering Speaker-Aware ASR, Real-Time TTS, and Speech-to-Speech Pipelines

!pip uninstall -y transformers -q
!pip install -q git+https://github.com/huggingface/transformers.git
!pip install -q torch torchaudio accelerate soundfile librosa scipy numpy
!pip install -q huggingface_hub ipywidgets gradio einops
!pip install -q flash-attn –no-build-isolation 2>/dev/null || echo “flash-attn optional”
!git clone -q –depth 1 https://github.com/microsoft/VibeVoice.git /content/VibeVoice 2>/dev/null || echo “Already cloned”
!pip install -q -e /content/VibeVoice

print(“=”*70)
print(“IMPORTANT: If this is your first run, restart the runtime now!”)
print(“Go to: Runtime -> Restart runtime, then run from CELL 2.”)
print(“=”*70)

import torch
import numpy as np
import soundfile as sf
import warnings
import sys
from IPython.display import Audio, display

warnings.filterwarnings(‘ignore’)
sys.path.insert(0, ‘/content/VibeVoice’)

import transformers
print(f”Transformers version: {transformers.__version__}”)

try:
from transformers import VibeVoiceAsrForConditionalGeneration
print(“VibeVoice ASR: Available”)
except ImportError:
print(“ERROR: VibeVoice not available. Please restart runtime and run Cell 1 again.”)
raise

SAMPLE_PODCAST = “https://huggingface.co/datasets/bezzam/vibevoice_samples/resolve/main/example_output/VibeVoice-1.5B_output.wav”
SAMPLE_GERMAN = “https://huggingface.co/datasets/bezzam/vibevoice_samples/resolve/main/realtime_model/vibevoice_tts_german.wav”

print(“Setup complete!”)

from transformers import AutoProcessor, VibeVoiceAsrForConditionalGeneration

print(“Loading VibeVoice ASR model (7B parameters)…”)
print(“First run downloads ~14GB – please wait…”)

asr_processor = AutoProcessor.from_pretrained(“microsoft/VibeVoice-ASR-HF”)
asr_model = VibeVoiceAsrForConditionalGeneration.from_pretrained(
“microsoft/VibeVoice-ASR-HF”,
device_map=”auto”,
torch_dtype=torch.float16,
)

print(f”ASR Model loaded on {asr_model.device}”)

def transcribe(audio_path, context=None, output_format=”parsed”):
inputs = asr_processor.apply_transcription_request(
audio=audio_path,
prompt=context,
).to(asr_model.device, asr_model.dtype)

output_ids = asr_model.generate(**inputs)
generated_ids = output_ids[:, inputs[“input_ids”].shape[1]:]
result = asr_processor.decode(generated_ids, return_format=output_format)[0]

return result

print(“=”*70)
print(“ASR DEMO: Podcast Transcription with Speaker Diarization”)
print(“=”*70)

print(“\nPlaying sample audio:”)
display(Audio(SAMPLE_PODCAST))

print(“\nTranscribing with speaker identification…”)
result = transcribe(SAMPLE_PODCAST, output_format=”parsed”)

print(“\nTRANSCRIPTION RESULTS:”)
print(“-“*70)
for segment in result:
speaker = segment[‘Speaker’]
start = segment[‘Start’]
end = segment[‘End’]
content = segment[‘Content’]
print(f”\n[Speaker {speaker}] {start:.2f}s – {end:.2f}s”)
print(f” {content}”)

print(“\n” + “=”*70)
print(“ASR DEMO: Context-Aware Transcription”)
print(“=”*70)

print(“\nComparing transcription WITH and WITHOUT context hotwords:”)
print(“-“*70)

result_no_ctx = transcribe(SAMPLE_GERMAN, context=None, output_format=”transcription_only”)
print(f”\nWITHOUT context: {result_no_ctx}”)

result_with_ctx = transcribe(SAMPLE_GERMAN, context=”About VibeVoice”, output_format=”transcription_only”)
print(f”WITH context: {result_with_ctx}”)

print(“\nNotice how ‘VibeVoice’ is recognized correctly when context is provided!”)

print(“\n” + “=”*70)
print(“ASR DEMO: Batch Processing”)
print(“=”*70)

audio_batch = [SAMPLE_GERMAN, SAMPLE_PODCAST]
prompts_batch = [“About VibeVoice”, None]

inputs = asr_processor.apply_transcription_request(
audio=audio_batch,
prompt=prompts_batch
).to(asr_model.device, asr_model.dtype)

output_ids = asr_model.generate(**inputs)
generated_ids = output_ids[:, inputs[“input_ids”].shape[1]:]
transcriptions = asr_processor.decode(generated_ids, return_format=”transcription_only”)

print(“\nBatch transcription results:”)
print(“-“*70)
for i, trans in enumerate(transcriptions):
preview = trans[:150] + “…” if len(trans) > 150 else trans
print(f”\nAudio {i+1}: {preview}”)

from transformers import AutoModelForCausalLM
from vibevoice.modular.modular_vibevoice_text_tokenizer import VibeVoiceTextTokenizerFast

print(“\n” + “=”*70)
print(“Loading VibeVoice Realtime TTS model (0.5B parameters)…”)
print(“=”*70)

tts_model = AutoModelForCausalLM.from_pretrained(
“microsoft/VibeVoice-Realtime-0.5B”,
trust_remote_code=True,
torch_dtype=torch.float16,
).to(“cuda” if torch.cuda.is_available() else “cpu”)

tts_tokenizer = VibeVoiceTextTokenizerFast.from_pretrained(“microsoft/VibeVoice-Realtime-0.5B”)
tts_model.set_ddpm_inference_steps(20)

print(f”TTS Model loaded on {next(tts_model.parameters()).device}”)

VOICES = [“Carter”, “Grace”, “Emma”, “Davis”]

def synthesize(text, voice=”Grace”, cfg_scale=3.0, steps=20, save_path=None):
tts_model.set_ddpm_inference_steps(steps)
input_ids = tts_tokenizer(text, return_tensors=”pt”).input_ids.to(tts_model.device)

output = tts_model.generate(
inputs=input_ids,
tokenizer=tts_tokenizer,
cfg_scale=cfg_scale,
return_speech=True,
show_progress_bar=True,
speaker_name=voice,
)

audio = output.audio.squeeze().cpu().numpy()
sample_rate = 24000

if save_path:
sf.write(save_path, audio, sample_rate)
print(f”Saved to: {save_path}”)

return audio, sample_rate

print(“\n” + “=”*70)
print(“TTS DEMO: Basic Speech Synthesis”)
print(“=”*70)

demo_texts = [
(“Hello! Welcome to VibeVoice, Microsoft’s open-source voice AI.”, “Grace”),
(“This model generates natural, expressive speech in real-time.”, “Carter”),
(“You can choose from multiple voice presets for different styles.”, “Emma”),
]

for text, voice in demo_texts:
print(f”\nText: {text}”)
print(f”Voice: {voice}”)
audio, sr = synthesize(text, voice=voice)
print(f”Duration: {len(audio)/sr:.2f} seconds”)
display(Audio(audio, rate=sr))

print(“\n” + “=”*70)
print(“TTS DEMO: Compare All Voice Presets”)
print(“=”*70)

comparison_text = “VibeVoice produces remarkably natural and expressive speech synthesis.”
print(f”\nSame text with different voices: \”{comparison_text}\”\n”)

for voice in VOICES:
print(f”Voice: {voice}”)
audio, sr = synthesize(comparison_text, voice=voice, steps=15)
display(Audio(audio, rate=sr))
print()

print(“\n” + “=”*70)
print(“TTS DEMO: Long-form Speech Generation”)
print(“=”*70)

long_text = “””
Welcome to today’s technology podcast! I’m excited to share the latest developments in artificial intelligence and speech synthesis.

Microsoft’s VibeVoice represents a breakthrough in voice AI. Unlike traditional text-to-speech systems, which struggle with long-form content, VibeVoice can generate coherent speech for extended durations.

The key innovation is the ultra-low frame-rate tokenizers operating at 7.5 hertz. This preserves audio quality while dramatically improving computational efficiency.

The system uses a next-token diffusion framework that combines a large language model for context understanding with a diffusion head for high-fidelity audio generation. This enables natural prosody, appropriate pauses, and expressive speech patterns.

Whether you’re building voice assistants, creating podcasts, or developing accessibility tools, VibeVoice offers a powerful foundation for your projects.

Thank you for listening!
“””

print(“Generating long-form speech (this takes a moment)…”)
audio, sr = synthesize(long_text.strip(), voice=”Carter”, cfg_scale=3.5, steps=25)
print(f”\nGenerated {len(audio)/sr:.2f} seconds of speech”)
display(Audio(audio, rate=sr))

sf.write(“/content/longform_output.wav”, audio, sr)
print(“Saved to: /content/longform_output.wav”)

print(“\n” + “=”*70)
print(“ADVANCED: Speech-to-Speech Pipeline”)
print(“=”*70)

print(“\nStep 1: Transcribing input audio…”)
transcription = transcribe(SAMPLE_GERMAN, context=”About VibeVoice”, output_format=”transcription_only”)
print(f”Transcription: {transcription}”)

response_text = f”I understood you said: {transcription} That’s a fascinating topic about AI technology!”

print(f”\nStep 2: Generating speech response…”)
print(f”Response: {response_text}”)

audio, sr = synthesize(response_text, voice=”Grace”, cfg_scale=3.0, steps=20)

print(f”\nStep 3: Playing generated response ({len(audio)/sr:.2f}s)”)
display(Audio(audio, rate=sr))

import gradio as gr

def tts_gradio(text, voice, cfg, steps):
if not text.strip():
return None
audio, sr = synthesize(text, voice=voice, cfg_scale=cfg, steps=int(steps))
return (sr, audio)

demo = gr.Interface(
fn=tts_gradio,
inputs=[
gr.Textbox(label=”Text to Synthesize”, lines=5,
value=”Hello! This is VibeVoice real-time text-to-speech.”),
gr.Dropdown(choices=VOICES, value=”Grace”, label=”Voice”),
gr.Slider(1.0, 5.0, value=3.0, step=0.5, label=”CFG Scale”),
gr.Slider(5, 50, value=20, step=5, label=”Inference Steps”),
],
outputs=gr.Audio(label=”Generated Speech”),
title=”VibeVoice Realtime TTS”,
description=”Generate natural speech from text using Microsoft’s VibeVoice model.”,
)

print(“\nLaunching interactive TTS interface…”)
demo.launch(share=True, quiet=True)

from google.colab import files
import os

print(“\n” + “=”*70)
print(“UPLOAD YOUR OWN AUDIO”)
print(“=”*70)

print(“\nUpload an audio file (wav, mp3, flac, etc.):”)
uploaded = files.upload()

if uploaded:
for filename, data in uploaded.items():
filepath = f”/content/{filename}”
with open(filepath, ‘wb’) as f:
f.write(data)

print(f”\nProcessing: {filename}”)
display(Audio(filepath))

result = transcribe(filepath, output_format=”parsed”)

print(“\nTranscription:”)
print(“-“*50)
if isinstance(result, list):
for seg in result:
print(f”[{seg.get(‘Start’,0):.2f}s-{seg.get(‘End’,0):.2f}s] Speaker {seg.get(‘Speaker’,0)}: {seg.get(‘Content’,”)}”)
else:
print(result)
else:
print(“No file uploaded – skipping this step”)

print(“\n” + “=”*70)
print(“MEMORY OPTIMIZATION TIPS”)
print(“=”*70)

print(“””
1. REDUCE ASR CHUNK SIZE (if out of memory with long audio):
output_ids = asr_model.generate(**inputs, acoustic_tokenizer_chunk_size=64000)

2. USE BFLOAT16 DTYPE:
model = VibeVoiceAsrForConditionalGeneration.from_pretrained(
model_id, torch_dtype=torch.bfloat16, device_map=”auto”)

3. REDUCE TTS INFERENCE STEPS (faster but lower quality):
tts_model.set_ddpm_inference_steps(10)

4. CLEAR GPU CACHE:
import gc
torch.cuda.empty_cache()
gc.collect()

5. GRADIENT CHECKPOINTING FOR TRAINING:
model.gradient_checkpointing_enable()
“””)

print(“\n” + “=”*70)
print(“DOWNLOAD GENERATED FILES”)
print(“=”*70)

output_files = [“/content/longform_output.wav”]

for filepath in output_files:
if os.path.exists(filepath):
print(f”Downloading: {os.path.basename(filepath)}”)
files.download(filepath)
else:
print(f”File not found: {filepath}”)

print(“\n” + “=”*70)
print(“TUTORIAL COMPLETE!”)
print(“=”*70)

print(“””
WHAT YOU LEARNED:

VIBEVOICE ASR (Speech-to-Text):
– 60-minute single-pass transcription
– Speaker diarization (who said what, when)
– Context-aware hotword recognition
– 50+ language support
– Batch processing

VIBEVOICE REALTIME TTS (Text-to-Speech):
– Real-time streaming (~300ms latency)
– Multiple voice presets
– Long-form generation (~10 minutes)
– Configurable quality/speed

RESOURCES:
GitHub: https://github.com/microsoft/VibeVoice
ASR Model: https://huggingface.co/microsoft/VibeVoice-ASR-HF
TTS Model: https://huggingface.co/microsoft/VibeVoice-Realtime-0.5B
ASR Paper: https://arxiv.org/pdf/2601.18184
TTS Paper: https://openreview.net/pdf?id=FihSkzyxdv

RESPONSIBLE USE:
– This is for research/development only
– Always disclose AI-generated content
– Do not use for impersonation or fraud
– Follow applicable laws and regulations
“””)