feat: add FunASR speech recognition backend

.github/backend-matrix.yml

@@ -326,6 +326,19 @@ include:
     dockerfile: "./backend/Dockerfile.python"
     context: "./"
     ubuntu-version: '2404'
+  - build-type: 'cublas'
+    cuda-major-version: "12"
+    cuda-minor-version: "8"
+    platforms: 'linux/amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-nvidia-cuda-12-funasr'
+    runs-on: 'ubuntu-latest'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "funasr"
+    dockerfile: "./backend/Dockerfile.python"
+    context: "./"
+    ubuntu-version: '2404'
   - build-type: 'cublas'
     cuda-major-version: "12"
     cuda-minor-version: "8"
@@ -973,6 +986,19 @@ include:
     dockerfile: "./backend/Dockerfile.python"
     context: "./"
     ubuntu-version: '2404'
+  - build-type: 'cublas'
+    cuda-major-version: "13"
+    cuda-minor-version: "0"
+    platforms: 'linux/amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-nvidia-cuda-13-funasr'
+    runs-on: 'ubuntu-latest'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "funasr"
+    dockerfile: "./backend/Dockerfile.python"
+    context: "./"
+    ubuntu-version: '2404'
   - build-type: 'cublas'
     cuda-major-version: "13"
     cuda-minor-version: "0"
@@ -1279,6 +1305,19 @@ include:
     backend: "qwen-asr"
     dockerfile: "./backend/Dockerfile.python"
     context: "./"
+  - build-type: 'l4t'
+    cuda-major-version: "13"
+    cuda-minor-version: "0"
+    platforms: 'linux/arm64'
+    tag-latest: 'auto'
+    tag-suffix: '-nvidia-l4t-cuda-13-arm64-funasr'
+    runs-on: 'ubuntu-24.04-arm'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    ubuntu-version: '2404'
+    backend: "funasr"
+    dockerfile: "./backend/Dockerfile.python"
+    context: "./"
   - build-type: 'l4t'
     cuda-major-version: "13"
     cuda-minor-version: "0"
@@ -2114,6 +2153,19 @@ include:
     dockerfile: "./backend/Dockerfile.python"
     context: "./"
     ubuntu-version: '2404'
+  - build-type: 'hipblas'
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-rocm-hipblas-funasr'
+    runs-on: 'ubuntu-latest'
+    base-image: "rocm/dev-ubuntu-24.04:7.2.1"
+    skip-drivers: 'false'
+    backend: "funasr"
+    dockerfile: "./backend/Dockerfile.python"
+    context: "./"
+    ubuntu-version: '2404'
   - build-type: 'hipblas'
     cuda-major-version: ""
     cuda-minor-version: ""
@@ -2366,6 +2418,19 @@ include:
     dockerfile: "./backend/Dockerfile.python"
     context: "./"
     ubuntu-version: '2204'
+  - build-type: 'l4t'
+    cuda-major-version: "12"
+    cuda-minor-version: "0"
+    platforms: 'linux/arm64'
+    tag-latest: 'auto'
+    tag-suffix: '-nvidia-l4t-funasr'
+    runs-on: 'ubuntu-24.04-arm'
+    base-image: "nvcr.io/nvidia/l4t-jetpack:r36.4.0"
+    skip-drivers: 'true'
+    backend: "funasr"
+    dockerfile: "./backend/Dockerfile.python"
+    context: "./"
+    ubuntu-version: '2204'
   - build-type: 'l4t'
     cuda-major-version: "12"
     cuda-minor-version: "0"
@@ -2575,6 +2640,19 @@ include:
     dockerfile: "./backend/Dockerfile.python"
     context: "./"
     ubuntu-version: '2404'
+  - build-type: 'intel'
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-intel-funasr'
+    runs-on: 'ubuntu-latest'
+    base-image: "intel/oneapi-basekit:2025.3.0-0-devel-ubuntu24.04"
+    skip-drivers: 'false'
+    backend: "funasr"
+    dockerfile: "./backend/Dockerfile.python"
+    context: "./"
+    ubuntu-version: '2404'
   - build-type: 'intel'
     cuda-major-version: ""
     cuda-minor-version: ""
@@ -5123,6 +5201,20 @@ include:
     dockerfile: "./backend/Dockerfile.python"
     context: "./"
     ubuntu-version: '2404'
+  - build-type: ''
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/amd64'
+    platform-tag: 'amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-cpu-funasr'
+    runs-on: 'ubuntu-latest'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "funasr"
+    dockerfile: "./backend/Dockerfile.python"
+    context: "./"
+    ubuntu-version: '2404'
   - build-type: ''
     cuda-major-version: ""
     cuda-minor-version: ""
@@ -5137,6 +5229,20 @@ include:
     dockerfile: "./backend/Dockerfile.python"
     context: "./"
     ubuntu-version: '2404'
+  - build-type: ''
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/arm64'
+    platform-tag: 'arm64'
+    tag-latest: 'auto'
+    tag-suffix: '-cpu-funasr'
+    runs-on: 'ubuntu-24.04-arm'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "funasr"
+    dockerfile: "./backend/Dockerfile.python"
+    context: "./"
+    ubuntu-version: '2404'
   - build-type: ''
     cuda-major-version: ""
     cuda-minor-version: ""
@@ -5480,6 +5586,9 @@ includeDarwin:
   - backend: "qwen-asr"
     tag-suffix: "-metal-darwin-arm64-qwen-asr"
     build-type: "mps"
+  - backend: "funasr"
+    tag-suffix: "-metal-darwin-arm64-funasr"
+    build-type: "mps"
   - backend: "nemo"
     tag-suffix: "-metal-darwin-arm64-nemo"
     build-type: "mps"

Makefile

@@ -1,5 +1,5 @@
 # Disable parallel execution for backend builds
-.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/outetts backends/piper backends/stablediffusion-ggml backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/longcat-video backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin
+.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/outetts backends/piper backends/stablediffusion-ggml backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/longcat-video backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/funasr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin
 
 GOCMD=go
 GOTEST=$(GOCMD) test
@@ -575,6 +575,7 @@ prepare-test-extra: protogen-python
 	$(MAKE) -C backend/python/fish-speech
 	$(MAKE) -C backend/python/faster-qwen3-tts
 	$(MAKE) -C backend/python/qwen-asr
+	$(MAKE) -C backend/python/funasr
 	$(MAKE) -C backend/python/nemo
 	$(MAKE) -C backend/python/voxcpm
 	$(MAKE) -C backend/python/faster-whisper
@@ -604,6 +605,7 @@ test-extra: prepare-test-extra
 	$(MAKE) -C backend/python/fish-speech test
 	$(MAKE) -C backend/python/faster-qwen3-tts test
 	$(MAKE) -C backend/python/qwen-asr test
+	$(MAKE) -C backend/python/funasr test
 	$(MAKE) -C backend/python/nemo test
 	$(MAKE) -C backend/python/voxcpm test
 	$(MAKE) -C backend/python/faster-whisper test
@@ -1268,6 +1270,7 @@ BACKEND_QWEN_TTS = qwen-tts|python|.|false|true
 BACKEND_FISH_SPEECH = fish-speech|python|.|false|true
 BACKEND_FASTER_QWEN3_TTS = faster-qwen3-tts|python|.|false|true
 BACKEND_QWEN_ASR = qwen-asr|python|.|false|true
+BACKEND_FUNASR = funasr|python|.|false|true
 BACKEND_NEMO = nemo|python|.|false|true
 BACKEND_VOXCPM = voxcpm|python|.|false|true
 BACKEND_WHISPERX = whisperx|python|.|false|true
@@ -1354,6 +1357,7 @@ $(eval $(call generate-docker-build-target,$(BACKEND_QWEN_TTS)))
 $(eval $(call generate-docker-build-target,$(BACKEND_FISH_SPEECH)))
 $(eval $(call generate-docker-build-target,$(BACKEND_FASTER_QWEN3_TTS)))
 $(eval $(call generate-docker-build-target,$(BACKEND_QWEN_ASR)))
+$(eval $(call generate-docker-build-target,$(BACKEND_FUNASR)))
 $(eval $(call generate-docker-build-target,$(BACKEND_NEMO)))
 $(eval $(call generate-docker-build-target,$(BACKEND_VOXCPM)))
 $(eval $(call generate-docker-build-target,$(BACKEND_WHISPERX)))
@@ -1379,7 +1383,7 @@ $(eval $(call generate-docker-build-target,$(BACKEND_SUPERTONIC)))
 docker-save-%: backend-images
 	docker save local-ai-backend:$* -o backend-images/$*.tar
 
-docker-build-backends: docker-build-llama-cpp docker-build-ik-llama-cpp docker-build-turboquant docker-build-ds4 docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-longcat-video docker-build-sglang docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-crispasr docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-liquid-audio docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-tinygrad docker-build-kokoros docker-build-sam3-cpp docker-build-rfdetr-cpp docker-build-qwen3-tts-cpp docker-build-omnivoice-cpp docker-build-vibevoice-cpp docker-build-localvqe docker-build-insightface docker-build-speaker-recognition docker-build-sherpa-onnx docker-build-cloud-proxy docker-build-supertonic docker-build-depth-anything-cpp docker-build-moss-transcribe-cpp docker-build-privacy-filter
+docker-build-backends: docker-build-llama-cpp docker-build-ik-llama-cpp docker-build-turboquant docker-build-ds4 docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-longcat-video docker-build-sglang docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-crispasr docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-liquid-audio docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-funasr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-tinygrad docker-build-kokoros docker-build-sam3-cpp docker-build-rfdetr-cpp docker-build-qwen3-tts-cpp docker-build-omnivoice-cpp docker-build-vibevoice-cpp docker-build-localvqe docker-build-insightface docker-build-speaker-recognition docker-build-sherpa-onnx docker-build-cloud-proxy docker-build-supertonic docker-build-depth-anything-cpp docker-build-moss-transcribe-cpp docker-build-privacy-filter
 
 ########################################################
 ### Mock Backend for E2E Tests

backend/README.md

@@ -47,7 +47,7 @@ The backend system provides language-specific Dockerfiles that handle the build
 - **mlx**: Apple Silicon optimization
 - **diffusers**: Stable Diffusion models
 - **longcat-video**: CUDA text/image-to-video and speech-driven avatar generation
-- **Audio**: coqui, faster-whisper, kitten-tts
+- **Audio**: coqui, faster-whisper, funasr, kitten-tts
 - **Vision**: mlx-vlm, rfdetr
 - **Specialized**: rerankers, chatterbox, kokoro
 

backend/index.yaml

@@ -1509,6 +1509,32 @@
     nvidia-l4t-cuda-12: "nvidia-l4t-faster-qwen3-tts"
     nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-faster-qwen3-tts"
   icon: https://cdn-avatars.huggingface.co/v1/production/uploads/620760a26e3b7210c2ff1943/-s1gyJfvbE1RgO5iBeNOi.png
+- &funasr
+  urls:
+    - https://github.com/modelscope/FunASR
+  description: |
+    FunASR is an industrial-grade speech recognition toolkit supporting 50+ languages.
+    Includes SenseVoice (170x realtime, emotion detection), Paraformer (highest Chinese
+    accuracy), and built-in VAD, punctuation restoration, and speaker diarization.
+  tags:
+    - speech-recognition
+    - ASR
+    - multilingual
+  license: mit
+  name: "funasr"
+  alias: "funasr"
+  capabilities:
+    nvidia: "cuda12-funasr"
+    intel: "intel-funasr"
+    amd: "rocm-funasr"
+    metal: "metal-funasr"
+    default: "cpu-funasr"
+    nvidia-cuda-13: "cuda13-funasr"
+    nvidia-cuda-12: "cuda12-funasr"
+    nvidia-l4t: "nvidia-l4t-funasr"
+    nvidia-l4t-cuda-12: "nvidia-l4t-funasr"
+    nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-funasr"
+  icon: https://avatars.githubusercontent.com/u/109454077
 - &qwen-asr
   urls:
     - https://github.com/QwenLM/Qwen3-ASR
@@ -6108,3 +6134,97 @@
   uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-supertonic"
   mirrors:
     - localai/localai-backends:master-metal-darwin-arm64-supertonic
+## funasr
+- !!merge <<: *funasr
+  name: "funasr-development"
+  capabilities:
+    nvidia: "cuda12-funasr-development"
+    intel: "intel-funasr-development"
+    amd: "rocm-funasr-development"
+    nvidia-l4t: "nvidia-l4t-funasr-development"
+    metal: "metal-funasr-development"
+    default: "cpu-funasr-development"
+    nvidia-cuda-13: "cuda13-funasr-development"
+    nvidia-cuda-12: "cuda12-funasr-development"
+    nvidia-l4t-cuda-12: "nvidia-l4t-funasr-development"
+    nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-funasr-development"
+- !!merge <<: *funasr
+  name: "cpu-funasr"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-funasr"
+  mirrors:
+    - localai/localai-backends:latest-cpu-funasr
+- !!merge <<: *funasr
+  name: "cpu-funasr-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-cpu-funasr"
+  mirrors:
+    - localai/localai-backends:master-cpu-funasr
+- !!merge <<: *funasr
+  name: "cuda12-funasr"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-funasr"
+  mirrors:
+    - localai/localai-backends:latest-gpu-nvidia-cuda-12-funasr
+- !!merge <<: *funasr
+  name: "cuda12-funasr-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-12-funasr"
+  mirrors:
+    - localai/localai-backends:master-gpu-nvidia-cuda-12-funasr
+- !!merge <<: *funasr
+  name: "cuda13-funasr"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-13-funasr"
+  mirrors:
+    - localai/localai-backends:latest-gpu-nvidia-cuda-13-funasr
+- !!merge <<: *funasr
+  name: "cuda13-funasr-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-funasr"
+  mirrors:
+    - localai/localai-backends:master-gpu-nvidia-cuda-13-funasr
+- !!merge <<: *funasr
+  name: "intel-funasr"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-intel-funasr"
+  mirrors:
+    - localai/localai-backends:latest-gpu-intel-funasr
+- !!merge <<: *funasr
+  name: "intel-funasr-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-intel-funasr"
+  mirrors:
+    - localai/localai-backends:master-gpu-intel-funasr
+- !!merge <<: *funasr
+  name: "rocm-funasr"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-rocm-hipblas-funasr"
+  mirrors:
+    - localai/localai-backends:latest-gpu-rocm-hipblas-funasr
+- !!merge <<: *funasr
+  name: "rocm-funasr-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-rocm-hipblas-funasr"
+  mirrors:
+    - localai/localai-backends:master-gpu-rocm-hipblas-funasr
+- !!merge <<: *funasr
+  name: "nvidia-l4t-funasr"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-funasr"
+  mirrors:
+    - localai/localai-backends:latest-nvidia-l4t-funasr
+- !!merge <<: *funasr
+  name: "nvidia-l4t-funasr-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-funasr"
+  mirrors:
+    - localai/localai-backends:master-nvidia-l4t-funasr
+- !!merge <<: *funasr
+  name: "cuda13-nvidia-l4t-arm64-funasr"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-cuda-13-arm64-funasr"
+  mirrors:
+    - localai/localai-backends:latest-nvidia-l4t-cuda-13-arm64-funasr
+- !!merge <<: *funasr
+  name: "cuda13-nvidia-l4t-arm64-funasr-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-cuda-13-arm64-funasr"
+  mirrors:
+    - localai/localai-backends:master-nvidia-l4t-cuda-13-arm64-funasr
+- !!merge <<: *funasr
+  name: "metal-funasr"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-funasr"
+  mirrors:
+    - localai/localai-backends:latest-metal-darwin-arm64-funasr
+- !!merge <<: *funasr
+  name: "metal-funasr-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-funasr"
+  mirrors:
+    - localai/localai-backends:master-metal-darwin-arm64-funasr

backend/python/README.md

@@ -20,6 +20,7 @@ The Python backends use a unified build system based on `libbackend.sh` that pro
 ### Audio & Speech
 - **coqui** - Coqui TTS models
 - **faster-whisper** - Fast Whisper speech recognition
+- **funasr** - Local multilingual transcription with FunASR and SenseVoice
 - **kitten-tts** - Lightweight TTS
 - **mlx-audio** - Apple Silicon audio processing
 - **chatterbox** - TTS model

backend/python/funasr/Makefile

@@ -0,0 +1,23 @@
+.PHONY: funasr
+funasr:
+	bash install.sh
+
+.PHONY: run
+run: funasr
+	@echo "Running funasr..."
+	bash run.sh
+	@echo "funasr run."
+
+.PHONY: test
+test: funasr
+	@echo "Testing funasr..."
+	bash test.sh
+	@echo "funasr tested."
+
+.PHONY: protogen-clean
+protogen-clean:
+	$(RM) backend_pb2_grpc.py backend_pb2.py
+
+.PHONY: clean
+clean: protogen-clean
+	rm -rf venv __pycache__