_server_app.py

"""FunASR Server — unified vLLM-based inference service.

Provides OpenAI-compatible API (/v1/audio/transcriptions) and REST API (/asr).
Uses vLLM for Fun-ASR-Nano (GPU) or AutoModel for supported models, including
MOSS-Transcribe-Diarize's joint transcription and anonymous speaker labels.
"""

import io
import os
import re
import time
import logging
import tempfile
from pathlib import Path
from typing import Iterable, Optional

import numpy as np
import soundfile as sf

try:
    from fastapi import FastAPI, UploadFile, File, Form, HTTPException
    from fastapi.middleware.cors import CORSMiddleware
    from fastapi.responses import JSONResponse
except ImportError:
    raise ImportError(
        "funasr-server requires additional packages. Install with: pip install vllm fastapi uvicorn python-multipart"
    )

logger = logging.getLogger("funasr.server")


PACKAGE_VERSION = (Path(__file__).resolve().parents[1] / "version.txt").read_text().strip()


_LANGUAGE_TAG_RE = re.compile(r"<\|(zh|en|yue|ja|ko)\|>")
MOSS_MODEL_REVISION = "e8681d68e7042738ffca8ac8212bc8fcb1131ab8"
NATIVE_DIARIZATION_MODELS = {"moss-transcribe-diarize"}
N8N_OPENAI_MODEL_ALIAS = "whisper-1"


def extract_language_from_asr_text(text):
    """Extract a SenseVoice language code before special tokens are removed."""
    if not isinstance(text, str):
        return None
    match = _LANGUAGE_TAG_RE.search(text)
    return match.group(1) if match else None


def resolve_transcription_language(requested_language, result):
    """Prefer the caller's language hint, then backend detection, else unknown."""
    if requested_language and requested_language.strip().lower() != "auto":
        return requested_language
    detected_language = result.get("language")
    if isinstance(detected_language, str) and detected_language:
        return detected_language
    return "unknown"


def resolve_openai_transcription_model(requested_model, default_model):
    """Map n8n's fixed OpenAI transcription model to the server default."""
    if requested_model == N8N_OPENAI_MODEL_ALIAS:
        return default_model
    return requested_model


def _split_text_for_openai_segments(text: str, max_chars: int = 80):
    """Split unsegmented ASR text into readable OpenAI-compatible cues."""
    text = text.strip()
    if not text:
        return []

    words = text.split()
    if not words:
        return [text[i : i + max_chars] for i in range(0, len(text), max_chars)]

    parts = []
    current = []
    current_len = 0
    for word in words:
        next_len = len(word) if not current else current_len + 1 + len(word)
        if current and next_len > max_chars:
            parts.append(" ".join(current))
            current = []
            current_len = 0

        current.append(word)
        current_len = len(word) if current_len == 0 else current_len + 1 + len(word)
        if word[-1:] in ".!?;:" and current_len >= max_chars // 2:
            parts.append(" ".join(current))
            current = []
            current_len = 0

    if current:
        parts.append(" ".join(current))

    if any(len(part) > max_chars for part in parts):
        return [text[i : i + max_chars] for i in range(0, len(text), max_chars)]

    return parts


def build_openai_fallback_segments(text: str, duration: float, max_chars: int = 80):
    """Build coarse timestamped segments when a backend returns text only."""
    parts = _split_text_for_openai_segments(text, max_chars=max_chars)
    if not parts:
        return []
    if len(parts) == 1 or duration <= 0:
        return [{"start": 0.0, "end": max(float(duration), 0.0), "text": parts[0]}]

    total_chars = sum(len(part) for part in parts)
    if total_chars <= 0:
        return [{"start": 0.0, "end": float(duration), "text": text.strip()}]

    segments = []
    consumed = 0
    previous_end = 0.0
    for i, part in enumerate(parts):
        consumed += len(part)
        end = float(duration) if i == len(parts) - 1 else float(duration) * consumed / total_chars
        end = max(end, previous_end)
        segments.append({"start": round(previous_end, 3), "end": round(end, 3), "text": part})
        previous_end = end

    return segments


def prepare_audio_for_inference(audio_data, sr, target_sr=16000):
    """Return mono float32 audio at target_sr for ASR inference."""
    audio_data = np.asarray(audio_data)
    if audio_data.ndim > 1:
        channel_axis = -1 if audio_data.shape[-1] <= audio_data.shape[0] else 0
        audio_data = audio_data.mean(axis=channel_axis)

    if sr != target_sr:
        import librosa
        audio_data = librosa.resample(audio_data, orig_sr=sr, target_sr=target_sr)
        sr = target_sr

    return audio_data.astype(np.float32), sr


def attach_speaker_labels(audio_data, sr, segments, speaker_model, device):
    """Run speaker diarization once and attach labels to timestamped segments."""
    if not segments:
        return segments

    import torch
    from funasr.models.campplus.cluster_backend import ClusterBackend
    from funasr.models.campplus.utils import distribute_spk, postprocess, sv_chunk

    audio_data, sr = prepare_audio_for_inference(audio_data, sr)
    duration = len(audio_data) / sr
    diarization_inputs = []
    segment_indexes = []
    for index, segment in enumerate(segments):
        start = max(float(segment.get("start", 0.0)), 0.0)
        end = min(max(float(segment.get("end", start)), start), duration)
        start_sample = int(start * sr)
        end_sample = int(end * sr)
        if end_sample <= start_sample:
            continue
        diarization_inputs.append([start, end, audio_data[start_sample:end_sample]])
        segment_indexes.append(index)

    chunks = sv_chunk(diarization_inputs, fs=sr)
    if not chunks:
        return segments

    speaker_results = speaker_model.generate(
        input=[chunk[2] for chunk in chunks], cache={}, is_final=True
    )
    embeddings = torch.cat(
        [speaker_result["spk_embedding"] for speaker_result in speaker_results], dim=0
    )
    labels = ClusterBackend(merge_thr=0.78).to(device)(embeddings.cpu(), oracle_num=None)
    if not isinstance(labels, np.ndarray):
        labels = np.asarray(labels)
    speaker_timeline = postprocess(
        sorted(chunks, key=lambda chunk: chunk[0]),
        None,
        labels,
        embeddings.detach().cpu().numpy(),
    )
    sentences = [
        {
            "text": segments[index]["text"],
            "start": int(float(segments[index]["start"]) * 1000),
            "end": int(float(segments[index]["end"]) * 1000),
        }
        for index in segment_indexes
    ]
    distribute_spk(sentences, speaker_timeline)
    for index, sentence in zip(segment_indexes, sentences):
        speaker = sentence.get("spk")
        if speaker is not None:
            segments[index]["speaker"] = f"SPK{speaker}"
    return segments


def build_openai_verbose_json(result, requested_language=None):
    """Build OpenAI-compatible verbose JSON while preserving FunASR extensions."""
    segments = []
    for index, segment in enumerate(result.get("segments", [])):
        item = {
            "id": index,
            "start": segment["start"],
            "end": segment["end"],
            "text": segment["text"],
            "words": segment.get("words", []),
        }
        if segment.get("speaker") is not None:
            item["speaker"] = segment["speaker"]
        segments.append(item)

    return {
        "task": "transcribe",
        "language": resolve_transcription_language(requested_language, result),
        "duration": result.get("duration", 0),
        "text": result["text"],
        "segments": segments,
    }


def create_app(
    device: str = "cuda",
    preload_model: str = "auto",
    model_path: str = None,
    hub: str = "ms",
    spk_model: str = "cam++",
    cors_origins: Optional[Iterable[str]] = None,
) -> FastAPI:
    if preload_model == "auto":
        preload_model = "fun-asr-nano" if device.startswith("cuda") else "sensevoice"

    app = FastAPI(title="FunASR Server", version=PACKAGE_VERSION)
    app.state.device = device
    app.state.engine = None
    app.state.vad_model = None
    app.state.spk_model = None
    app.state.spk_model_name = spk_model
    app.state.fallback_models = {}
    app.state.model_path = model_path
    app.state.hub = hub
    app.state.openai_transcription_model = "custom" if model_path else preload_model

    normalized_origins = []
    for origin in cors_origins or []:
        origin = origin.strip()
        if origin and origin not in normalized_origins:
            normalized_origins.append(origin)
    if normalized_origins:
        app.add_middleware(
            CORSMiddleware,
            allow_origins=normalized_origins,
            allow_credentials=False,
            allow_methods=["GET", "POST", "OPTIONS"],
            allow_headers=["Authorization", "Content-Type"],
        )

    # Non-LLM model configs (use AutoModel, no vLLM)
    FALLBACK_CONFIGS = {
        "sensevoice": {
            "model": "iic/SenseVoiceSmall",
            "vad_model": "fsmn-vad",
            "vad_kwargs": {"max_single_segment_time": 30000},
        },
        "paraformer": {
            "model": "paraformer-zh",
            "vad_model": "fsmn-vad",
            "punc_model": "ct-punc",
        },
        "moss-transcribe-diarize": {
            "model": "OpenMOSS-Team/MOSS-Transcribe-Diarize",
            "model_revision": MOSS_MODEL_REVISION,
            "hub": "hf",
            "backend": "hf",
            "trust_remote_code": True,
        },
    }

    def _load_spk_model():
        """Lazily load diarization only when a request opts in with spk=true."""
        if app.state.spk_model is not None:
            return app.state.spk_model
        if not app.state.spk_model_name:
            raise HTTPException(400, "Speaker diarization is disabled; configure --spk-model")

        from funasr import AutoModel

        logger.info(f"Loading speaker model: {app.state.spk_model_name}")
        app.state.spk_model = AutoModel(
            model=app.state.spk_model_name,
            device=device,
            disable_update=True,
        )
        logger.info("Speaker model ready.")
        return app.state.spk_model

    def _load_vllm_engine():
        """Load Fun-ASR-Nano vLLM engine. Falls back to AutoModel if vLLM unavailable."""
        if app.state.engine is not None or "fun-asr-nano" in app.state.fallback_models:
            return
        try:
            from funasr.models.fun_asr_nano.inference_vllm import FunASRNanoVLLM
            from funasr import AutoModel as _AutoModel

            logger.info("Loading Fun-ASR-Nano vLLM engine...")
            t0 = time.time()
            # Use custom model_path if provided, otherwise default. In both
            # cases, honor the server-level hub selection.
            vllm_model = app.state.model_path if app.state.model_path else "FunAudioLLM/Fun-ASR-Nano-2512"
            vllm_hub = app.state.hub
            engine = FunASRNanoVLLM.from_pretrained(
                model=vllm_model,
                hub=vllm_hub,
                device=device,
                dtype="bf16",
                max_model_len=4096,
                gpu_memory_utilization=0.5,
            )
            logger.info(f"vLLM engine ready in {time.time()-t0:.1f}s")

            logger.info("Loading VAD model...")
            vad_model = _AutoModel(model="fsmn-vad", device=device, disable_update=True)
            app.state.engine = engine
            app.state.vad_model = vad_model
            app.state.use_vllm = True
            logger.info("VAD ready.")
        except Exception as e:
            logger.warning(f"vLLM failed ({e}), falling back to AutoModel for fun-asr-nano")
            app.state.use_vllm = False
            from funasr import AutoModel
            cfg = {
                "model": app.state.model_path if app.state.model_path else "FunAudioLLM/Fun-ASR-Nano-2512",
                "hub": app.state.hub,
                "trust_remote_code": True,
                "vad_model": "fsmn-vad",
                "vad_kwargs": {"max_single_segment_time": 30000},
                "device": device,
                "disable_update": True,
            }
            app.state.fallback_models["fun-asr-nano"] = AutoModel(**cfg)
            logger.info(f"Fallback AutoModel loaded for fun-asr-nano with model={cfg['model']}, hub={cfg['hub']}.")

    def _load_fallback(name: str):
        """Load non-LLM model via AutoModel."""
        if name in app.state.fallback_models:
            return app.state.fallback_models[name]
        if name not in FALLBACK_CONFIGS and not app.state.model_path:
            return None
        from funasr import AutoModel
        cfg = FALLBACK_CONFIGS.get(name, {}).copy()
        # Override with custom model_path and hub if provided
        if app.state.model_path:
            cfg["model"] = app.state.model_path
            cfg["hub"] = app.state.hub
        elif app.state.hub and "hub" not in cfg:
            cfg["hub"] = app.state.hub
        cfg["device"] = device
        cfg["disable_update"] = True
        logger.info(f"Loading fallback model '{name}' with model={cfg['model']}, hub={cfg['hub']}...")
        model = AutoModel(**cfg)
        app.state.fallback_models[name] = model
        return model

    def _process_vllm(audio_data, sr, language=None, hotwords=None, use_spk=False):
        """Process audio with vLLM engine (Fun-ASR-Nano)."""
        audio_data, sr = prepare_audio_for_inference(audio_data, sr)

        # VAD
        vad_res = app.state.vad_model.generate(input=audio_data, fs=sr)
        segments = vad_res[0]["value"] if vad_res and vad_res[0].get("value") else [[0, int(len(audio_data)*1000/sr)]]

        seg_audios = []
        seg_times = []
        for seg in segments:
            s0 = int(seg[0] * sr / 1000)
            s1 = int(seg[1] * sr / 1000)
            seg_audio = audio_data[s0:s1]
            if len(seg_audio) > sr * 0.3:
                seg_audios.append(seg_audio)
                seg_times.append((seg[0], seg[1]))

        if not seg_audios:
            return {"text": "", "segments": [], "duration": len(audio_data)/sr}

        # repetition_penalty is left at the neutral 1.0: the Fun-ASR-Nano vLLM
        # engine runs in prompt-embeds mode, where any other value crashes the
        # CUDA kernel (see issue #2948 and fun_asr_nano.vllm_utils).
        gen_kwargs = {"max_new_tokens": 500, "repetition_penalty": 1.0}
        if language:
            gen_kwargs["language"] = language
        if hotwords:
            gen_kwargs["hotwords"] = hotwords

        results = app.state.engine.generate(inputs=seg_audios, **gen_kwargs)

        output_segments = []
        full_text_parts = []
        for r, (start_ms, end_ms) in zip(results, seg_times):
            text = r["text"]
            seg_info = {"text": text, "start": start_ms/1000, "end": end_ms/1000}
            if "timestamps" in r:
                offset = start_ms / 1000
                seg_info["words"] = [
                    {"word": ts["token"], "start": ts["start_time"]+offset, "end": ts["end_time"]+offset}
                    for ts in r["timestamps"]
                ]
            output_segments.append(seg_info)
            full_text_parts.append(text)

        if use_spk:
            attach_speaker_labels(
                audio_data,
                sr,
                output_segments,
                _load_spk_model(),
                device,
            )

        return {
            "text": "".join(full_text_parts),
            "segments": output_segments,
            "duration": len(audio_data) / sr,
        }

    def _process_fallback(model_name, audio_path, language=None, use_spk=False):
        """Process with non-LLM model (SenseVoice/Paraformer)."""
        model = _load_fallback(model_name)
        try:
            duration = float(sf.info(audio_path).duration)
        except Exception:
            duration = 0.0
        kwargs = {"input": audio_path, "batch_size": 1}
        if language:
            kwargs["language"] = language
        result = model.generate(**kwargs)
        raw_text = result[0]["text"]
        detected_language = extract_language_from_asr_text(raw_text)
        text = re.sub(r'<\|[^|]*\|>', '', raw_text).strip()
        segments = []
        if "sentence_info" in result[0]:
            for s in result[0]["sentence_info"]:
                segment = {
                    "start": s.get("start", 0)/1000,
                    "end": s.get("end", 0)/1000,
                    "text": re.sub(
                        r'<\|[^|]*\|>', '', s.get("text") or s.get("sentence", "")
                    ).strip(),
                }
                if s.get("spk") is not None:
                    segment["speaker"] = s["spk"]
                segments.append(segment)
        if not segments and text:
            segments = build_openai_fallback_segments(text, duration)
        if use_spk and model_name not in NATIVE_DIARIZATION_MODELS and segments:
            audio_data, sr = sf.read(audio_path)
            attach_speaker_labels(
                audio_data,
                sr,
                segments,
                _load_spk_model(),
                device,
            )
        return {
            "text": text,
            "segments": segments,
            "duration": duration,
            "language": detected_language,
        }

    # Pre-load
    if app.state.model_path:
        # When custom model_path is provided, use it as the model name for loading
        logger.info(f"Loading custom model: {app.state.model_path} (hub: {app.state.hub})")
        _load_fallback("custom")
    elif preload_model == "fun-asr-nano":
        _load_vllm_engine()
    else:
        _load_fallback(preload_model)

    @app.post("/v1/audio/transcriptions")
    async def transcribe(
        file: UploadFile = File(...),
        model: str = Form(default="fun-asr-nano"),
        language: Optional[str] = Form(default=None),
        response_format: Optional[str] = Form(default="json"),
        spk: bool = Form(default=False),
    ):
        content = await file.read()
        t0 = time.perf_counter()
        model = resolve_openai_transcription_model(
            model, app.state.openai_transcription_model
        )

        if model == "fun-asr-nano":
            _load_vllm_engine()
            if app.state.use_vllm:
                audio_data, sr = sf.read(io.BytesIO(content))
                result = _process_vllm(audio_data, sr, language=language, use_spk=spk)
            else:
                suffix = os.path.splitext(file.filename)[1] if file.filename else ".wav"
                with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp:
                    tmp.write(content)
                    tmp_path = tmp.name
                try:
                    result = _process_fallback(
                        "fun-asr-nano", tmp_path, language=language, use_spk=spk
                    )
                finally:
                    os.unlink(tmp_path)
        elif model in FALLBACK_CONFIGS or model == "custom":
            suffix = os.path.splitext(file.filename)[1] if file.filename else ".wav"
            with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp:
                tmp.write(content)
                tmp_path = tmp.name
            try:
                result = _process_fallback(
                    model, tmp_path, language=language, use_spk=spk
                )
            finally:
                os.unlink(tmp_path)
        else:
            available = ["fun-asr-nano", "custom"] + list(FALLBACK_CONFIGS.keys())
            raise HTTPException(400, f"Unknown model '{model}'. Available: {', '.join(available)}")

        t1 = time.perf_counter()

        if response_format == "verbose_json":
            return JSONResponse(build_openai_verbose_json(result, requested_language=language))
        elif response_format == "text":
            return JSONResponse(result["text"])
        else:
            return JSONResponse({"text": result["text"]})

    @app.post("/asr")
    async def asr_endpoint(
        file: UploadFile = File(...),
        language: Optional[str] = Form(default=None),
        hotwords: str = Form(default=""),
        spk: bool = Form(default=False),
    ):
        """Full-featured ASR endpoint with timestamps and speaker diarization."""
        content = await file.read()
        _load_vllm_engine()
        hw_list = [w.strip() for w in hotwords.split(",") if w.strip()] if hotwords else None

        t0 = time.perf_counter()
        if app.state.use_vllm:
            audio_data, sr = sf.read(io.BytesIO(content))
            result = _process_vllm(audio_data, sr, language=language, hotwords=hw_list, use_spk=spk)
        else:
            suffix = ".wav"
            with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp:
                tmp.write(content)
                tmp_path = tmp.name
            try:
                result = _process_fallback(
                    "fun-asr-nano", tmp_path, language=language, use_spk=spk
                )
            finally:
                os.unlink(tmp_path)
        t1 = time.perf_counter()

        result["processing_time"] = round(t1 - t0, 3)
        result["rtf"] = round((t1 - t0) / result["duration"], 4) if result.get("duration", 0) > 0 else 0
        return JSONResponse(result)

    @app.get("/v1/models")
    async def list_models():
        all_models = ["fun-asr-nano"] + list(FALLBACK_CONFIGS.keys())
        if app.state.model_path:
            all_models.append("custom")
        return JSONResponse({"object": "list", "data": [{"id": n, "object": "model"} for n in all_models]})

    @app.get("/health")
    async def health():
        loaded = []
        if app.state.engine is not None:
            loaded.append("fun-asr-nano (vLLM)")
        loaded.extend(app.state.fallback_models.keys())
        return {"status": "ok", "device": device, "models_loaded": loaded}

    return app