Whisper: Transcription, faster-whisper, VAD and Diarization Reference Guide

# Pick ONE runtime — they all read the same .pt weights but ship different APIs

# 1 · Reference implementation (PyTorch, slow but matches paper)
pip install -U openai-whisper

# 2 · faster-whisper — CTranslate2 backend, 4× faster on CPU/GPU
pip install faster-whisper

# 3 · Hugging Face transformers — pipeline / Trainer / fine-tuning
pip install transformers accelerate

# 4 · whisper.cpp — C++ build, CPU + Apple Metal, no Python required
brew install whisper-cpp           # or build from source

# ffmpeg is required by every Python runtime
brew install ffmpeg                # macOS
apt install ffmpeg                 # Debian / Ubuntu

import whisper

# Loads weights to ~/.cache/whisper/ on first call
model = whisper.load_model("base", device="cuda")  # cpu, cuda, mps

result = model.transcribe(
    "interview.mp3",
    language="en",          # pin language to skip detection
    task="transcribe",      # or "translate" → English
    fp16=True,              # half precision on GPU
    temperature=0.0,        # deterministic
    no_speech_threshold=0.6,
    beam_size=5,
    word_timestamps=True,
)

print(result["text"])
for seg in result["segments"]:
    print(f"[{seg['start']:.2f} → {seg['end']:.2f}] {seg['text']}")

from faster_whisper import WhisperModel

# int8 = best speed/quality tradeoff on CPU; "float16" on GPU
model = WhisperModel(
    "large-v3",
    device="cuda",
    compute_type="float16",          # int8, int8_float16, float16, float32
)

segments, info = model.transcribe(
    "interview.mp3",
    language="en",
    beam_size=5,
    vad_filter=True,                 # built-in Silero VAD
    vad_parameters={"min_silence_duration_ms": 500},
    word_timestamps=True,
)

print(f"Detected: {info.language} ({info.language_probability:.2%})")

# segments is a generator — iterate to actually run inference
for seg in segments:
    print(f"[{seg.start:.2f} → {seg.end:.2f}] {seg.text}")
    for w in seg.words or []:
        print(f"  {w.start:.2f}-{w.end:.2f}: {w.word}")

from transformers import pipeline
import torch

asr = pipeline(
    "automatic-speech-recognition",
    model="openai/whisper-large-v3",
    torch_dtype=torch.float16,
    device="cuda",
    chunk_length_s=30,                   # long-form support
    batch_size=8,                        # batched chunks
    return_timestamps="word",            # "word" | True (segment) | False
    generate_kwargs={"language": "en", "task": "transcribe"},
)

result = asr("interview.mp3")
print(result["text"])
for chunk in result["chunks"][:5]:
    print(chunk["timestamp"], chunk["text"])

from faster_whisper import WhisperModel
from datetime import timedelta

def fmt(t: float) -> str:
    td = timedelta(seconds=t)
    s = str(td)
    return ("0:" + s if td.seconds < 3600 else s).replace(".", ",")[:11]

model = WhisperModel("medium", device="cuda", compute_type="float16")

segments, info = model.transcribe(
    "lecture.mp4",                    # ffmpeg extracts audio automatically
    vad_filter=True,
    word_timestamps=True,
    initial_prompt="Lecture on retrieval-augmented generation, May 2026.",
)

with open("lecture.srt", "w", encoding="utf-8") as f:
    for i, seg in enumerate(segments, start=1):
        f.write(f"{i}\n")
        f.write(f"{fmt(seg.start)} --> {fmt(seg.end)}\n")
        f.write(seg.text.strip() + "\n\n")

print(f"Wrote subtitles · language: {info.language}")