feat: multimodal attachments — file→text, transcription, native image/audio/video (+ CLI @path//attach)

agentmcp/server.go

@@ -6,13 +6,14 @@ import (
 	"sync"
 
 	"github.com/modelcontextprotocol/go-sdk/mcp"
+	"github.com/mudler/nib/chat"
 )
 
 // session is the slice of *chat.Session the MCP server needs. An interface so
 // tests can drive converse/interrupt without a live LLM. *chat.Session
 // satisfies it.
 type session interface {
-	SendMessage(text string) (string, error)
+	SendMessage(text string, parts ...chat.ContentPart) (string, error)
 	InjectUser(msg string) bool
 	RunLive() bool
 	Interrupt()

agentmcp/server_test.go

@@ -36,7 +36,7 @@ type fakeSession struct {
 	interrupt   int
 }
 
-func (f *fakeSession) SendMessage(text string) (string, error) {
+func (f *fakeSession) SendMessage(text string, parts ...chat.ContentPart) (string, error) {
 	if f.parkFirst {
 		f.cb.OnParked("working on it")
 	}

attachments/apply.go

@@ -0,0 +1,94 @@
+package attachments
+
+import (
+	"context"
+	"fmt"
+	"path/filepath"
+	"strings"
+
+	"github.com/mudler/nib/specialist"
+)
+
+type Extractor func(path string) (string, error)
+type Transcriber func(ctx context.Context, path string) (string, error)
+
+type Part struct {
+	Kind    Kind
+	DataURI string
+	// Format is the container/codec derived from the file extension (e.g. "wav",
+	// "mp3", "m4a"). Only populated for natively-sent audio (SendAsAudio), where
+	// it feeds cogito's input_audio.format wire field; empty for image/video.
+	Format string
+}
+type Blocked struct {
+	Path   string
+	Reason string
+}
+type Result struct {
+	Parts        []Part
+	TextPreamble string
+	Blocked      []Blocked
+}
+
+func label(path, treatment, text string) string {
+	base := filepath.Base(path)
+	return fmt.Sprintf("[Attached: %s — %s]\n%s\n[End of %s]\n\n", base, treatment, text, base)
+}
+
+// Apply resolves each file to its treatment and executes it, producing the native
+// parts + text preamble SendMessage needs, plus any blocked files for the caller
+// to surface. overrides maps a file path to an audio override (may be nil).
+//
+// Apply is all-or-nothing on hard failures: the first extractor, transcriber, or
+// DataURI error aborts the whole batch and is returned as-is (the caller is
+// expected to resolve or remove the offending file and retry). Only capability
+// Block outcomes are non-fatal — they are collected into Result.Blocked rather
+// than erroring, so a batch containing a lone unsupported file still returns a
+// nil error with that file recorded in Blocked.
+func Apply(ctx context.Context, files []string, caps ModelCapabilities, overrides map[string]Override,
+	extract Extractor, transcribe Transcriber) (Result, error) {
+	var res Result
+	for _, path := range files {
+		kind := Sniff(path)
+		ov := OverrideNone
+		if overrides != nil {
+			ov = overrides[path]
+		}
+		switch Route(kind, caps, ov) {
+		case SendAsImage:
+			uri, err := specialist.DataURI(path)
+			if err != nil {
+				return res, err
+			}
+			res.Parts = append(res.Parts, Part{Kind: KindImage, DataURI: uri})
+		case SendAsAudio:
+			uri, err := specialist.DataURI(path)
+			if err != nil {
+				return res, err
+			}
+			format := strings.TrimPrefix(strings.ToLower(filepath.Ext(path)), ".")
+			res.Parts = append(res.Parts, Part{Kind: KindAudio, DataURI: uri, Format: format})
+		case SendAsVideo:
+			uri, err := specialist.DataURI(path)
+			if err != nil {
+				return res, err
+			}
+			res.Parts = append(res.Parts, Part{Kind: KindVideo, DataURI: uri})
+		case Transcribe:
+			text, err := transcribe(ctx, path)
+			if err != nil {
+				return res, err
+			}
+			res.TextPreamble += label(path, "transcribed", text)
+		case ConvertToText:
+			text, err := extract(path)
+			if err != nil {
+				return res, err
+			}
+			res.TextPreamble += label(path, "converted to text", text)
+		case Block:
+			res.Blocked = append(res.Blocked, Blocked{Path: path, Reason: "active model can't accept this file type"})
+		}
+	}
+	return res, nil
+}

attachments/apply_test.go

@@ -0,0 +1,121 @@
+package attachments
+
+import (
+	"context"
+	"errors"
+	"os"
+	"strings"
+	"testing"
+)
+
+func TestApply(t *testing.T) {
+	extract := func(path string) (string, error) { return "DOC TEXT", nil }
+	transcribe := func(_ context.Context, path string) (string, error) { return "SPOKEN", nil }
+
+	// data URI needs real files for image; fake the image via a temp file
+	dir := t.TempDir()
+	img := dir + "/pic.png"
+	if err := os.WriteFile(img, []byte("\x89PNGdata"), 0o644); err != nil {
+		t.Fatalf("write temp image: %v", err)
+	}
+
+	files := []string{img, "notes.pdf", "memo.m4a"}
+	caps := ModelCapabilities{InputModalities: []string{"text", "image"}} // vision, no audio
+	res, err := Apply(context.Background(), files, caps, nil, extract, transcribe)
+	if err != nil {
+		t.Fatalf("Apply: %v", err)
+	}
+	if len(res.Parts) != 1 || res.Parts[0].Kind != KindImage {
+		t.Fatalf("expected 1 image part, got %+v", res.Parts)
+	}
+	if !strings.Contains(res.TextPreamble, "DOC TEXT") || !strings.Contains(res.TextPreamble, "SPOKEN") {
+		t.Fatalf("preamble missing doc/transcript: %q", res.TextPreamble)
+	}
+	if !strings.Contains(res.TextPreamble, "[Attached: notes.pdf") {
+		t.Fatalf("preamble missing label: %q", res.TextPreamble)
+	}
+}
+
+// TestApplyAudioCarriesFormat verifies that a SendAsAudio outcome populates the
+// Part.Format from the file extension (not the mime), so cogito's
+// input_audio.format wire field is well-formed.
+func TestApplyAudioCarriesFormat(t *testing.T) {
+	extract := func(path string) (string, error) { return "DOC TEXT", nil }
+	transcribe := func(_ context.Context, path string) (string, error) { return "SPOKEN", nil }
+
+	// specialist.DataURI reads the file, so it must exist on disk.
+	dir := t.TempDir()
+	aud := dir + "/x.wav"
+	if err := os.WriteFile(aud, []byte("RIFFdata"), 0o644); err != nil {
+		t.Fatalf("write temp audio: %v", err)
+	}
+
+	caps := ModelCapabilities{InputModalities: []string{"text", "audio"}} // native audio
+	res, err := Apply(context.Background(), []string{aud}, caps, nil, extract, transcribe)
+	if err != nil {
+		t.Fatalf("Apply: %v", err)
+	}
+	if len(res.Parts) != 1 || res.Parts[0].Kind != KindAudio {
+		t.Fatalf("expected 1 audio part, got %+v", res.Parts)
+	}
+	if res.Parts[0].Format != "wav" {
+		t.Fatalf("expected audio Format %q, got %q", "wav", res.Parts[0].Format)
+	}
+}
+
+// TestApplyBlockedAccumulates verifies that a capability Block is non-fatal:
+// a lone unsupported image on a text-only model is recorded in Result.Blocked,
+// Apply returns a nil error, and neither extractor nor transcriber is invoked.
+func TestApplyBlockedAccumulates(t *testing.T) {
+	extract := func(path string) (string, error) {
+		t.Fatalf("extract must not be called for a blocked image: %s", path)
+		return "", nil
+	}
+	transcribe := func(_ context.Context, path string) (string, error) {
+		t.Fatalf("transcribe must not be called for a blocked image: %s", path)
+		return "", nil
+	}
+
+	// Real temp file so Sniff→Route sees KindImage (Block needs no DataURI read).
+	dir := t.TempDir()
+	img := dir + "/pic.png"
+	if err := os.WriteFile(img, []byte("\x89PNGdata"), 0o644); err != nil {
+		t.Fatalf("write temp image: %v", err)
+	}
+
+	caps := ModelCapabilities{InputModalities: []string{"text"}} // text-only → image blocked
+	res, err := Apply(context.Background(), []string{img}, caps, nil, extract, transcribe)
+	if err != nil {
+		t.Fatalf("Apply: unexpected error: %v", err)
+	}
+	if len(res.Parts) != 0 {
+		t.Fatalf("expected no parts, got %+v", res.Parts)
+	}
+	if len(res.Blocked) != 1 {
+		t.Fatalf("expected 1 blocked file, got %+v", res.Blocked)
+	}
+	if res.Blocked[0].Path != img {
+		t.Fatalf("expected blocked path %q, got %q", img, res.Blocked[0].Path)
+	}
+}
+
+// TestApplyExtractErrorAborts verifies Apply's fail-fast contract: the first
+// extractor error aborts the batch, is returned to the caller, and nothing
+// partial leaks into the returned Result.
+func TestApplyExtractErrorAborts(t *testing.T) {
+	extract := func(_ string) (string, error) { return "", errors.New("boom") }
+	transcribe := func(_ context.Context, _ string) (string, error) { return "SPOKEN", nil }
+
+	// notes.pdf routes to ConvertToText; the fake extractor runs, so no real file needed.
+	caps := ModelCapabilities{InputModalities: []string{"text"}}
+	res, err := Apply(context.Background(), []string{"notes.pdf"}, caps, nil, extract, transcribe)
+	if err == nil {
+		t.Fatalf("expected error from failing extractor, got nil")
+	}
+	if len(res.Parts) != 0 {
+		t.Fatalf("fail-fast: expected no parts, got %+v", res.Parts)
+	}
+	if res.TextPreamble != "" {
+		t.Fatalf("fail-fast: expected empty preamble, got %q", res.TextPreamble)
+	}
+}

attachments/capabilities.go

@@ -0,0 +1,48 @@
+package attachments
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+)
+
+func textOnly() ModelCapabilities { return ModelCapabilities{InputModalities: []string{"text"}} }
+
+// FetchCapabilities asks LocalAI's /models/capabilities endpoint for the given
+// model's input modalities. Any failure (network, decode, unknown model) is
+// treated as text-only so we never hand media to a model that can't take it.
+func FetchCapabilities(ctx context.Context, baseURL, apiKey, model string) ModelCapabilities {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, baseURL+"/models/capabilities", nil)
+	if err != nil {
+		return textOnly()
+	}
+	if apiKey != "" {
+		req.Header.Set("Authorization", "Bearer "+apiKey)
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return textOnly()
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return textOnly()
+	}
+	var body struct {
+		Data []struct {
+			ID              string   `json:"id"`
+			InputModalities []string `json:"input_modalities"`
+		} `json:"data"`
+	}
+	if err := json.NewDecoder(resp.Body).Decode(&body); err != nil {
+		return textOnly()
+	}
+	for _, m := range body.Data {
+		if m.ID == model {
+			if len(m.InputModalities) == 0 {
+				return textOnly()
+			}
+			return ModelCapabilities{InputModalities: m.InputModalities}
+		}
+	}
+	return textOnly()
+}

attachments/capabilities_test.go

@@ -0,0 +1,31 @@
+package attachments
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func TestFetchCapabilities(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = w.Write([]byte(`{"object":"list","data":[
+			{"id":"vlm","input_modalities":["text","image"]},
+			{"id":"omni","input_modalities":["text","image","audio"]}]}`))
+	}))
+	defer srv.Close()
+
+	got := FetchCapabilities(context.Background(), srv.URL, "", "omni")
+	if !got.Accepts("audio") || !got.Accepts("image") {
+		t.Fatalf("omni should accept audio+image, got %v", got.InputModalities)
+	}
+}
+
+func TestFetchCapabilitiesFailSafe(t *testing.T) {
+	// unreachable server URL → fail-safe to text-only
+	got := FetchCapabilities(context.Background(), "http://127.0.0.1:0", "", "whatever")
+	if got.Accepts("image") || !got.Accepts("text") {
+		t.Fatalf("fail-safe must be text-only, got %v", got.InputModalities)
+	}
+}

attachments/kind.go

@@ -0,0 +1,43 @@
+package attachments
+
+import (
+	"path/filepath"
+	"strings"
+
+	"github.com/mudler/nib/extraction"
+)
+
+type Kind int
+
+const (
+	KindImage Kind = iota
+	KindAudio
+	KindVideo
+	KindDocument
+	KindText
+	KindUnknown
+)
+
+var imageExts = map[string]bool{".png": true, ".jpg": true, ".jpeg": true, ".gif": true, ".webp": true, ".bmp": true}
+var audioExts = map[string]bool{".mp3": true, ".wav": true, ".ogg": true, ".m4a": true, ".flac": true, ".mpeg": true, ".mpga": true}
+var videoExts = map[string]bool{".mp4": true, ".avi": true, ".mkv": true, ".mov": true, ".webm": true, ".m4v": true}
+
+// Sniff classifies a file by extension. Documents/text delegate to the
+// extraction package's predicates so the two stay in sync.
+func Sniff(path string) Kind {
+	ext := strings.ToLower(filepath.Ext(path))
+	switch {
+	case imageExts[ext]:
+		return KindImage
+	case audioExts[ext]:
+		return KindAudio
+	case videoExts[ext]:
+		return KindVideo
+	case extraction.IsDocumentFile(path):
+		return KindDocument
+	case extraction.IsPlainTextFile(path):
+		return KindText
+	default:
+		return KindUnknown
+	}
+}