diff --git a/agentmcp/server.go b/agentmcp/server.go index 1984f95..4f7de2a 100644 --- a/agentmcp/server.go +++ b/agentmcp/server.go @@ -6,13 +6,14 @@ import ( "sync" "github.com/modelcontextprotocol/go-sdk/mcp" + "github.com/mudler/nib/chat" ) // session is the slice of *chat.Session the MCP server needs. An interface so // tests can drive converse/interrupt without a live LLM. *chat.Session // satisfies it. type session interface { - SendMessage(text string) (string, error) + SendMessage(text string, parts ...chat.ContentPart) (string, error) InjectUser(msg string) bool RunLive() bool Interrupt() diff --git a/agentmcp/server_test.go b/agentmcp/server_test.go index 4fc684d..7986dbd 100644 --- a/agentmcp/server_test.go +++ b/agentmcp/server_test.go @@ -36,7 +36,7 @@ type fakeSession struct { interrupt int } -func (f *fakeSession) SendMessage(text string) (string, error) { +func (f *fakeSession) SendMessage(text string, parts ...chat.ContentPart) (string, error) { if f.parkFirst { f.cb.OnParked("working on it") } diff --git a/attachments/apply.go b/attachments/apply.go new file mode 100644 index 0000000..8f54019 --- /dev/null +++ b/attachments/apply.go @@ -0,0 +1,94 @@ +package attachments + +import ( + "context" + "fmt" + "path/filepath" + "strings" + + "github.com/mudler/nib/specialist" +) + +type Extractor func(path string) (string, error) +type Transcriber func(ctx context.Context, path string) (string, error) + +type Part struct { + Kind Kind + DataURI string + // Format is the container/codec derived from the file extension (e.g. "wav", + // "mp3", "m4a"). Only populated for natively-sent audio (SendAsAudio), where + // it feeds cogito's input_audio.format wire field; empty for image/video. + Format string +} +type Blocked struct { + Path string + Reason string +} +type Result struct { + Parts []Part + TextPreamble string + Blocked []Blocked +} + +func label(path, treatment, text string) string { + base := filepath.Base(path) + return fmt.Sprintf("[Attached: %s — %s]\n%s\n[End of %s]\n\n", base, treatment, text, base) +} + +// Apply resolves each file to its treatment and executes it, producing the native +// parts + text preamble SendMessage needs, plus any blocked files for the caller +// to surface. overrides maps a file path to an audio override (may be nil). +// +// Apply is all-or-nothing on hard failures: the first extractor, transcriber, or +// DataURI error aborts the whole batch and is returned as-is (the caller is +// expected to resolve or remove the offending file and retry). Only capability +// Block outcomes are non-fatal — they are collected into Result.Blocked rather +// than erroring, so a batch containing a lone unsupported file still returns a +// nil error with that file recorded in Blocked. +func Apply(ctx context.Context, files []string, caps ModelCapabilities, overrides map[string]Override, + extract Extractor, transcribe Transcriber) (Result, error) { + var res Result + for _, path := range files { + kind := Sniff(path) + ov := OverrideNone + if overrides != nil { + ov = overrides[path] + } + switch Route(kind, caps, ov) { + case SendAsImage: + uri, err := specialist.DataURI(path) + if err != nil { + return res, err + } + res.Parts = append(res.Parts, Part{Kind: KindImage, DataURI: uri}) + case SendAsAudio: + uri, err := specialist.DataURI(path) + if err != nil { + return res, err + } + format := strings.TrimPrefix(strings.ToLower(filepath.Ext(path)), ".") + res.Parts = append(res.Parts, Part{Kind: KindAudio, DataURI: uri, Format: format}) + case SendAsVideo: + uri, err := specialist.DataURI(path) + if err != nil { + return res, err + } + res.Parts = append(res.Parts, Part{Kind: KindVideo, DataURI: uri}) + case Transcribe: + text, err := transcribe(ctx, path) + if err != nil { + return res, err + } + res.TextPreamble += label(path, "transcribed", text) + case ConvertToText: + text, err := extract(path) + if err != nil { + return res, err + } + res.TextPreamble += label(path, "converted to text", text) + case Block: + res.Blocked = append(res.Blocked, Blocked{Path: path, Reason: "active model can't accept this file type"}) + } + } + return res, nil +} diff --git a/attachments/apply_test.go b/attachments/apply_test.go new file mode 100644 index 0000000..8e84842 --- /dev/null +++ b/attachments/apply_test.go @@ -0,0 +1,121 @@ +package attachments + +import ( + "context" + "errors" + "os" + "strings" + "testing" +) + +func TestApply(t *testing.T) { + extract := func(path string) (string, error) { return "DOC TEXT", nil } + transcribe := func(_ context.Context, path string) (string, error) { return "SPOKEN", nil } + + // data URI needs real files for image; fake the image via a temp file + dir := t.TempDir() + img := dir + "/pic.png" + if err := os.WriteFile(img, []byte("\x89PNGdata"), 0o644); err != nil { + t.Fatalf("write temp image: %v", err) + } + + files := []string{img, "notes.pdf", "memo.m4a"} + caps := ModelCapabilities{InputModalities: []string{"text", "image"}} // vision, no audio + res, err := Apply(context.Background(), files, caps, nil, extract, transcribe) + if err != nil { + t.Fatalf("Apply: %v", err) + } + if len(res.Parts) != 1 || res.Parts[0].Kind != KindImage { + t.Fatalf("expected 1 image part, got %+v", res.Parts) + } + if !strings.Contains(res.TextPreamble, "DOC TEXT") || !strings.Contains(res.TextPreamble, "SPOKEN") { + t.Fatalf("preamble missing doc/transcript: %q", res.TextPreamble) + } + if !strings.Contains(res.TextPreamble, "[Attached: notes.pdf") { + t.Fatalf("preamble missing label: %q", res.TextPreamble) + } +} + +// TestApplyAudioCarriesFormat verifies that a SendAsAudio outcome populates the +// Part.Format from the file extension (not the mime), so cogito's +// input_audio.format wire field is well-formed. +func TestApplyAudioCarriesFormat(t *testing.T) { + extract := func(path string) (string, error) { return "DOC TEXT", nil } + transcribe := func(_ context.Context, path string) (string, error) { return "SPOKEN", nil } + + // specialist.DataURI reads the file, so it must exist on disk. + dir := t.TempDir() + aud := dir + "/x.wav" + if err := os.WriteFile(aud, []byte("RIFFdata"), 0o644); err != nil { + t.Fatalf("write temp audio: %v", err) + } + + caps := ModelCapabilities{InputModalities: []string{"text", "audio"}} // native audio + res, err := Apply(context.Background(), []string{aud}, caps, nil, extract, transcribe) + if err != nil { + t.Fatalf("Apply: %v", err) + } + if len(res.Parts) != 1 || res.Parts[0].Kind != KindAudio { + t.Fatalf("expected 1 audio part, got %+v", res.Parts) + } + if res.Parts[0].Format != "wav" { + t.Fatalf("expected audio Format %q, got %q", "wav", res.Parts[0].Format) + } +} + +// TestApplyBlockedAccumulates verifies that a capability Block is non-fatal: +// a lone unsupported image on a text-only model is recorded in Result.Blocked, +// Apply returns a nil error, and neither extractor nor transcriber is invoked. +func TestApplyBlockedAccumulates(t *testing.T) { + extract := func(path string) (string, error) { + t.Fatalf("extract must not be called for a blocked image: %s", path) + return "", nil + } + transcribe := func(_ context.Context, path string) (string, error) { + t.Fatalf("transcribe must not be called for a blocked image: %s", path) + return "", nil + } + + // Real temp file so Sniff→Route sees KindImage (Block needs no DataURI read). + dir := t.TempDir() + img := dir + "/pic.png" + if err := os.WriteFile(img, []byte("\x89PNGdata"), 0o644); err != nil { + t.Fatalf("write temp image: %v", err) + } + + caps := ModelCapabilities{InputModalities: []string{"text"}} // text-only → image blocked + res, err := Apply(context.Background(), []string{img}, caps, nil, extract, transcribe) + if err != nil { + t.Fatalf("Apply: unexpected error: %v", err) + } + if len(res.Parts) != 0 { + t.Fatalf("expected no parts, got %+v", res.Parts) + } + if len(res.Blocked) != 1 { + t.Fatalf("expected 1 blocked file, got %+v", res.Blocked) + } + if res.Blocked[0].Path != img { + t.Fatalf("expected blocked path %q, got %q", img, res.Blocked[0].Path) + } +} + +// TestApplyExtractErrorAborts verifies Apply's fail-fast contract: the first +// extractor error aborts the batch, is returned to the caller, and nothing +// partial leaks into the returned Result. +func TestApplyExtractErrorAborts(t *testing.T) { + extract := func(_ string) (string, error) { return "", errors.New("boom") } + transcribe := func(_ context.Context, _ string) (string, error) { return "SPOKEN", nil } + + // notes.pdf routes to ConvertToText; the fake extractor runs, so no real file needed. + caps := ModelCapabilities{InputModalities: []string{"text"}} + res, err := Apply(context.Background(), []string{"notes.pdf"}, caps, nil, extract, transcribe) + if err == nil { + t.Fatalf("expected error from failing extractor, got nil") + } + if len(res.Parts) != 0 { + t.Fatalf("fail-fast: expected no parts, got %+v", res.Parts) + } + if res.TextPreamble != "" { + t.Fatalf("fail-fast: expected empty preamble, got %q", res.TextPreamble) + } +} diff --git a/attachments/capabilities.go b/attachments/capabilities.go new file mode 100644 index 0000000..6cba2a1 --- /dev/null +++ b/attachments/capabilities.go @@ -0,0 +1,48 @@ +package attachments + +import ( + "context" + "encoding/json" + "net/http" +) + +func textOnly() ModelCapabilities { return ModelCapabilities{InputModalities: []string{"text"}} } + +// FetchCapabilities asks LocalAI's /models/capabilities endpoint for the given +// model's input modalities. Any failure (network, decode, unknown model) is +// treated as text-only so we never hand media to a model that can't take it. +func FetchCapabilities(ctx context.Context, baseURL, apiKey, model string) ModelCapabilities { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, baseURL+"/models/capabilities", nil) + if err != nil { + return textOnly() + } + if apiKey != "" { + req.Header.Set("Authorization", "Bearer "+apiKey) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return textOnly() + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return textOnly() + } + var body struct { + Data []struct { + ID string `json:"id"` + InputModalities []string `json:"input_modalities"` + } `json:"data"` + } + if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { + return textOnly() + } + for _, m := range body.Data { + if m.ID == model { + if len(m.InputModalities) == 0 { + return textOnly() + } + return ModelCapabilities{InputModalities: m.InputModalities} + } + } + return textOnly() +} diff --git a/attachments/capabilities_test.go b/attachments/capabilities_test.go new file mode 100644 index 0000000..b58f8db --- /dev/null +++ b/attachments/capabilities_test.go @@ -0,0 +1,31 @@ +package attachments + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" +) + +func TestFetchCapabilities(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"object":"list","data":[ + {"id":"vlm","input_modalities":["text","image"]}, + {"id":"omni","input_modalities":["text","image","audio"]}]}`)) + })) + defer srv.Close() + + got := FetchCapabilities(context.Background(), srv.URL, "", "omni") + if !got.Accepts("audio") || !got.Accepts("image") { + t.Fatalf("omni should accept audio+image, got %v", got.InputModalities) + } +} + +func TestFetchCapabilitiesFailSafe(t *testing.T) { + // unreachable server URL → fail-safe to text-only + got := FetchCapabilities(context.Background(), "http://127.0.0.1:0", "", "whatever") + if got.Accepts("image") || !got.Accepts("text") { + t.Fatalf("fail-safe must be text-only, got %v", got.InputModalities) + } +} diff --git a/attachments/kind.go b/attachments/kind.go new file mode 100644 index 0000000..859abb4 --- /dev/null +++ b/attachments/kind.go @@ -0,0 +1,43 @@ +package attachments + +import ( + "path/filepath" + "strings" + + "github.com/mudler/nib/extraction" +) + +type Kind int + +const ( + KindImage Kind = iota + KindAudio + KindVideo + KindDocument + KindText + KindUnknown +) + +var imageExts = map[string]bool{".png": true, ".jpg": true, ".jpeg": true, ".gif": true, ".webp": true, ".bmp": true} +var audioExts = map[string]bool{".mp3": true, ".wav": true, ".ogg": true, ".m4a": true, ".flac": true, ".mpeg": true, ".mpga": true} +var videoExts = map[string]bool{".mp4": true, ".avi": true, ".mkv": true, ".mov": true, ".webm": true, ".m4v": true} + +// Sniff classifies a file by extension. Documents/text delegate to the +// extraction package's predicates so the two stay in sync. +func Sniff(path string) Kind { + ext := strings.ToLower(filepath.Ext(path)) + switch { + case imageExts[ext]: + return KindImage + case audioExts[ext]: + return KindAudio + case videoExts[ext]: + return KindVideo + case extraction.IsDocumentFile(path): + return KindDocument + case extraction.IsPlainTextFile(path): + return KindText + default: + return KindUnknown + } +} diff --git a/attachments/route.go b/attachments/route.go new file mode 100644 index 0000000..2f0a5cf --- /dev/null +++ b/attachments/route.go @@ -0,0 +1,56 @@ +package attachments + +import "slices" + +type ModelCapabilities struct { + InputModalities []string // "text","image","audio","video" +} + +func (m ModelCapabilities) Accepts(mod string) bool { return slices.Contains(m.InputModalities, mod) } + +type Override int + +const ( + OverrideNone Override = iota + OverrideTranscribe +) + +type Treatment int + +const ( + SendAsImage Treatment = iota + SendAsAudio + SendAsVideo + Transcribe + ConvertToText + Block +) + +// Route is pure: it maps a file kind + the active model's capabilities + an +// optional (audio-only) override to a treatment. See the spec's decision table. +func Route(kind Kind, caps ModelCapabilities, override Override) Treatment { + switch kind { + case KindImage: + if caps.Accepts("image") { + return SendAsImage + } + return Block + case KindAudio: + if override == OverrideTranscribe { + return Transcribe + } + if caps.Accepts("audio") { + return SendAsAudio + } + return Transcribe // text-only model → parakeet + case KindVideo: + if caps.Accepts("video") { + return SendAsVideo + } + return Block // Phase 1: no extract-audio fallback yet + case KindDocument, KindText: + return ConvertToText + default: + return ConvertToText // best-effort: try to read unknown as text + } +} diff --git a/attachments/route_test.go b/attachments/route_test.go new file mode 100644 index 0000000..7c00256 --- /dev/null +++ b/attachments/route_test.go @@ -0,0 +1,41 @@ +package attachments + +import "testing" + +func caps(mods ...string) ModelCapabilities { return ModelCapabilities{InputModalities: mods} } + +func TestRoute(t *testing.T) { + cases := []struct { + name string + kind Kind + caps ModelCapabilities + override Override + want Treatment + }{ + {"image to vision", KindImage, caps("text", "image"), OverrideNone, SendAsImage}, + {"image to text-only", KindImage, caps("text"), OverrideNone, Block}, + {"audio to omni", KindAudio, caps("text", "audio"), OverrideNone, SendAsAudio}, + {"audio override transcribe", KindAudio, caps("text", "audio"), OverrideTranscribe, Transcribe}, + {"audio to text-only", KindAudio, caps("text"), OverrideNone, Transcribe}, + {"video to text-only", KindVideo, caps("text"), OverrideNone, Block}, + {"video to video model", KindVideo, caps("text", "video"), OverrideNone, SendAsVideo}, + {"document any", KindDocument, caps("text"), OverrideNone, ConvertToText}, + {"text any", KindText, caps("text"), OverrideNone, ConvertToText}, + } + for _, tc := range cases { + if got := Route(tc.kind, tc.caps, tc.override); got != tc.want { + t.Errorf("%s: Route=%d want %d", tc.name, got, tc.want) + } + } +} + +func TestSniff(t *testing.T) { + for ext, want := range map[string]Kind{ + "a.png": KindImage, "b.jpg": KindImage, "c.m4a": KindAudio, "d.wav": KindAudio, + "e.mp4": KindVideo, "f.pdf": KindDocument, "g.docx": KindDocument, "h.md": KindText, + } { + if got := Sniff(ext); got != want { + t.Errorf("Sniff(%s)=%d want %d", ext, got, want) + } + } +} diff --git a/attachstage/attachstage.go b/attachstage/attachstage.go new file mode 100644 index 0000000..c56edee --- /dev/null +++ b/attachstage/attachstage.go @@ -0,0 +1,31 @@ +// Package attachstage holds the pure logic for combining CLI/TUI staged +// attachments with inline @path files from a slash action. It lives in its own +// package so both the cmd (CLI REPL) and tui frontends can reuse BuildSend +// without a cmd→tui dependency. +package attachstage + +import ( + "github.com/mudler/nib/attachments" + "github.com/mudler/nib/slash" +) + +// StagedFile is a file staged via /attach, awaiting the next send. +type StagedFile struct { + Path string + Transcribe bool +} + +// BuildSend combines staged files (first) with the action's inline @path files, +// and builds the transcribe-only override map. +func BuildSend(pending []StagedFile, action slash.Action) ([]string, map[string]attachments.Override) { + files := make([]string, 0, len(pending)+len(action.Files)) + overrides := map[string]attachments.Override{} + for _, s := range pending { + files = append(files, s.Path) + if s.Transcribe { + overrides[s.Path] = attachments.OverrideTranscribe + } + } + files = append(files, action.Files...) + return files, overrides +} diff --git a/attachstage/attachstage_test.go b/attachstage/attachstage_test.go new file mode 100644 index 0000000..08f3e87 --- /dev/null +++ b/attachstage/attachstage_test.go @@ -0,0 +1,24 @@ +package attachstage + +import ( + "testing" + + "github.com/mudler/nib/attachments" + "github.com/mudler/nib/slash" +) + +func TestBuildSend(t *testing.T) { + pending := []StagedFile{{Path: "a.wav", Transcribe: true}, {Path: "b.png"}} + action := slash.Action{Kind: slash.KindSend, Text: "go", Files: []string{"c.pdf"}} + files, overrides := BuildSend(pending, action) + // staged first, then inline @path + if len(files) != 3 || files[0] != "a.wav" || files[1] != "b.png" || files[2] != "c.pdf" { + t.Fatalf("combine order wrong: %v", files) + } + if overrides["a.wav"] != attachments.OverrideTranscribe { + t.Fatalf("a.wav must carry transcribe override: %v", overrides) + } + if _, ok := overrides["b.png"]; ok { + t.Fatalf("non-transcribe staged file must not be in overrides") + } +} diff --git a/chat/attach.go b/chat/attach.go new file mode 100644 index 0000000..ed8e1e7 --- /dev/null +++ b/chat/attach.go @@ -0,0 +1,61 @@ +package chat + +import ( + "context" + + "github.com/mudler/nib/attachments" + "github.com/mudler/nib/extraction" + "github.com/mudler/nib/specialist" +) + +// composeAttachments merges the preamble ahead of the user text and maps +// attachments.Part → chat.ContentPart. Pure/testable. +func composeAttachments(userText string, res attachments.Result) (string, []ContentPart) { + text := res.TextPreamble + userText + var parts []ContentPart + for _, p := range res.Parts { + k := PartImage + audioFormat := "" + switch p.Kind { + case attachments.KindAudio: + k = PartAudio + audioFormat = p.Format // feeds input_audio.format; empty for image/video + case attachments.KindVideo: + k = PartVideo + } + parts = append(parts, ContentPart{Kind: k, DataURI: p.DataURI, AudioFormat: audioFormat}) + } + return text, parts +} + +// SendWithAttachments resolves treatments for files against the active model's +// capabilities, runs conversion/transcription, and sends one multimodal turn. +// Returns any blocked files (media the active model can't accept) for the caller +// to surface. +func (s *Session) SendWithAttachments(ctx context.Context, text string, files []string, + overrides map[string]attachments.Override) (string, []attachments.Blocked, error) { + + caps := attachments.FetchCapabilities(ctx, s.baseURL, s.apiKey, s.llmModel) + sp := specialist.New(s.baseURL, s.apiKey) + + extract := func(path string) (string, error) { + if extraction.IsPlainTextFile(path) { + return extraction.ReadTextFile(path) + } + return extraction.ExtractText(path) + } + transcribe := func(ctx context.Context, path string) (string, error) { + return sp.Transcribe(ctx, path, s.transcribeModel) + } + + res, err := attachments.Apply(ctx, files, caps, overrides, extract, transcribe) + if err != nil { + return "", nil, err + } + finalText, parts := composeAttachments(text, res) + if finalText == "" && len(parts) == 0 { + return "", res.Blocked, nil // nothing sendable (all blocked, no text) + } + reply, err := s.SendMessage(finalText, parts...) + return reply, res.Blocked, err +} diff --git a/chat/attach_test.go b/chat/attach_test.go new file mode 100644 index 0000000..4729ca6 --- /dev/null +++ b/chat/attach_test.go @@ -0,0 +1,34 @@ +package chat + +import ( + "strings" + "testing" + + "github.com/mudler/nib/attachments" +) + +func TestComposeAttachments(t *testing.T) { + res := attachments.Result{ + TextPreamble: "[Attached: a.pdf — converted to text]\nBODY\n[End of a.pdf]\n\n", + Parts: []attachments.Part{ + {Kind: attachments.KindImage, DataURI: "data:image/png;base64,AA"}, + {Kind: attachments.KindVideo, DataURI: "data:video/mp4;base64,BB"}, + {Kind: attachments.KindAudio, DataURI: "data:audio/wav;base64,AA", Format: "wav"}, + }, + } + text, parts := composeAttachments("summarize", res) + if !strings.HasPrefix(text, "[Attached: a.pdf") || !strings.HasSuffix(text, "summarize") { + t.Fatalf("preamble must precede user text: %q", text) + } + if len(parts) != 3 || parts[0].Kind != PartImage || parts[1].Kind != PartVideo || parts[2].Kind != PartAudio { + t.Fatalf("expected image+video+audio ContentParts, got %+v", parts) + } + // audio Part.Format must propagate to ContentPart.AudioFormat (input_audio.format). + if parts[2].AudioFormat != "wav" { + t.Fatalf("expected audio AudioFormat %q, got %q", "wav", parts[2].AudioFormat) + } + // image/video carry no audio format. + if parts[0].AudioFormat != "" || parts[1].AudioFormat != "" { + t.Fatalf("image/video must have empty AudioFormat, got %q %q", parts[0].AudioFormat, parts[1].AudioFormat) + } +} diff --git a/chat/contentpart.go b/chat/contentpart.go new file mode 100644 index 0000000..d90bcd2 --- /dev/null +++ b/chat/contentpart.go @@ -0,0 +1,60 @@ +package chat + +import ( + "encoding/base64" + "strings" + + "github.com/mudler/cogito" +) + +type PartKind int + +const ( + PartImage PartKind = iota + PartAudio + PartVideo +) + +// ContentPart is nib's native multimodal part, satisfying cogito.TypedMultimedia +// so SendMessage can hand image/audio/video parts to the fragment. DataURI is a +// base64 data: URI (e.g. data:image/png;base64,...). AudioFormat is the audio +// container (e.g. "wav") used for the input_audio wire form. +type ContentPart struct { + Kind PartKind + DataURI string + AudioFormat string +} + +// URL returns the data URI (used for image_url / video_url parts). +func (p ContentPart) URL() string { return p.DataURI } + +// Format returns the audio container for input_audio; "" for image/video. +func (p ContentPart) Format() string { return p.AudioFormat } + +// MediaKind maps to cogito's media kind. +func (p ContentPart) MediaKind() cogito.MediaKind { + switch p.Kind { + case PartAudio: + return cogito.MediaAudio + case PartVideo: + return cogito.MediaVideo + default: + return cogito.MediaImage + } +} + +// Data returns the raw base64 payload (no data: prefix) for input_audio; "" +// for image/video (which travel via URL()). +func (p ContentPart) Data() string { + if p.Kind != PartAudio { + return "" + } + if i := strings.Index(p.DataURI, ";base64,"); i >= 0 { + return p.DataURI[i+len(";base64,"):] + } + // Fallback: if DataURI is already raw base64, validate & return as-is. + if _, err := base64.StdEncoding.DecodeString(p.DataURI); err == nil { + return p.DataURI + } + return "" +} diff --git a/chat/contentpart_test.go b/chat/contentpart_test.go new file mode 100644 index 0000000..10b0885 --- /dev/null +++ b/chat/contentpart_test.go @@ -0,0 +1,35 @@ +package chat + +import ( + "testing" + + "github.com/mudler/cogito" +) + +func TestContentPartImplementsTypedMultimedia(t *testing.T) { + var _ cogito.TypedMultimedia = ContentPart{} + + a := ContentPart{Kind: PartAudio, DataURI: "data:audio/wav;base64,AA", AudioFormat: "wav"} + if a.MediaKind() != cogito.MediaAudio { + t.Fatalf("audio → cogito.MediaAudio, got %v", a.MediaKind()) + } + if a.URL() != "data:audio/wav;base64,AA" || a.Format() != "wav" { + t.Fatalf("URL/Format mismatch: %q %q", a.URL(), a.Format()) + } + + v := ContentPart{Kind: PartVideo, DataURI: "data:video/mp4;base64,BB"} + if v.MediaKind() != cogito.MediaVideo { + t.Fatalf("video → cogito.MediaVideo, got %v", v.MediaKind()) + } + i := ContentPart{Kind: PartImage, DataURI: "data:image/png;base64,CC"} + if i.MediaKind() != cogito.MediaImage { + t.Fatalf("image → cogito.MediaImage, got %v", i.MediaKind()) + } + // audio carries base64 via Data(); image/video via URL() + if a.Data() != "AA" { + t.Fatalf("audio Data() must be the base64 payload %q, got %q", "AA", a.Data()) + } + if i.Data() != "" { + t.Fatalf("image Data() must be empty (image travels via URL()), got %q", i.Data()) + } +} diff --git a/chat/session.go b/chat/session.go index 092b317..bd67cf8 100644 --- a/chat/session.go +++ b/chat/session.go @@ -99,6 +99,8 @@ type Session struct { llmModel string apiKey string baseURL string + transcribeModel string + visionModel string metadata map[string]string // global per-request metadata; merged with per-agent overrides reasoningEffort string // OpenAI reasoning_effort sent on every request (e.g. "none") @@ -238,6 +240,8 @@ func NewSession(ctx context.Context, cfg types.Config, callbacks Callbacks, tran llmModel: cfg.Model, apiKey: cfg.APIKey, baseURL: cfg.BaseURL, + transcribeModel: cfg.TranscribeModel, + visionModel: cfg.VisionModel, metadata: cfg.Metadata, reasoningEffort: cfg.ReasoningEffort, mcpClient: client, @@ -672,7 +676,21 @@ func (s *Session) mcpToolFilter() func(*mcp.ClientSession, string) bool { } } -func (s *Session) SendMessage(text string) (string, error) { +// buildUserFragment appends the user turn to the fragment, attaching multimodal +// parts. cogito's AddMessage routes image parts into image_url MultiContent and +// audio/video into the fragment's transient PendingNativeParts (send-once). +func buildUserFragment(f cogito.Fragment, text string, parts []ContentPart) cogito.Fragment { + if len(parts) == 0 { + return f.AddMessage("user", text) + } + mm := make([]cogito.Multimedia, 0, len(parts)) + for _, p := range parts { + mm = append(mm, p) // ContentPart implements cogito.TypedMultimedia + } + return f.AddMessage("user", text, mm...) +} + +func (s *Session) SendMessage(text string, parts ...ContentPart) (string, error) { if s.hooks != nil { s.hooks.Fire(s.ctx, hooks.EventUserPromptSubmit, "", map[string]any{"event": "UserPromptSubmit", "prompt": text}) } @@ -715,7 +733,7 @@ func (s *Session) SendMessage(text string) (string, error) { if s.systemPrompt != "" { s.fragment = s.fragment.AddMessage("system", s.systemPrompt) } - s.fragment = s.fragment.AddMessage("user", text) + s.fragment = buildUserFragment(s.fragment, text, parts) s.messages = append(s.messages, openai.ChatCompletionMessage{ Role: "user", Content: text, diff --git a/chat/session_multimodal_test.go b/chat/session_multimodal_test.go new file mode 100644 index 0000000..701caf9 --- /dev/null +++ b/chat/session_multimodal_test.go @@ -0,0 +1,37 @@ +package chat + +import ( + "testing" + + "github.com/mudler/cogito" +) + +func TestBuildUserFragmentMultimodal(t *testing.T) { + f := cogito.Fragment{} + f = buildUserFragment(f, "look", []ContentPart{ + {Kind: PartImage, DataURI: "data:image/png;base64,AA"}, + {Kind: PartAudio, DataURI: "data:audio/wav;base64,BB", AudioFormat: "wav"}, + }) + last := f.Messages[len(f.Messages)-1] + // image → image_url in MultiContent; audio → PendingNativeParts (cogito send-once) + imageParts := 0 + for _, p := range last.MultiContent { + if p.Type == "image_url" { + imageParts++ + } + } + if imageParts != 1 { + t.Fatalf("expected 1 image_url part, got %d", imageParts) + } + if len(f.PendingNativeParts) != 1 || f.PendingNativeParts[0].Kind != cogito.MediaAudio { + t.Fatalf("expected 1 pending audio native part, got %+v", f.PendingNativeParts) + } +} + +func TestBuildUserFragmentTextOnly(t *testing.T) { + f := buildUserFragment(cogito.Fragment{}, "hi", nil) + last := f.Messages[len(f.Messages)-1] + if last.Content != "hi" || len(last.MultiContent) != 0 { + t.Fatalf("text-only must use scalar Content, got %+v", last) + } +} diff --git a/cmd/cli.go b/cmd/cli.go index f9c40b6..17803c1 100644 --- a/cmd/cli.go +++ b/cmd/cli.go @@ -5,11 +5,14 @@ import ( "context" "fmt" "os" + "path/filepath" "strings" "sync" "time" "github.com/modelcontextprotocol/go-sdk/mcp" + "github.com/mudler/nib/attachments" + "github.com/mudler/nib/attachstage" "github.com/mudler/nib/chat" wizmcp "github.com/mudler/nib/mcp" "github.com/mudler/nib/slash" @@ -260,6 +263,10 @@ func RunCLI(ctx context.Context, cfg types.Config, shellJobs *wizmcp.ShellJobs, // Display help immediately help() + // Files staged via /attach, sent with the next message and cleared on + // successful send only. + var pending []attachstage.StagedFile + for { select { case <-ctx.Done(): @@ -312,11 +319,47 @@ func RunCLI(ctx context.Context, cfg types.Config, shellJobs *wizmcp.ShellJobs, fmt.Println(theme.Subtle.Render(compactNotice(before, after))) } continue + case slash.KindAttach: + switch action.AttachOp { + case slash.AttachStage: + pending = append(pending, attachstage.StagedFile{Path: action.AttachPath, Transcribe: action.Transcribe}) + mode := "default" + if action.Transcribe { + mode = "transcribe" + } + fmt.Println(theme.Subtle.Render("attached: " + filepath.Base(action.AttachPath) + " (" + mode + ") — sends with your next message")) + case slash.AttachList: + if len(pending) == 0 { + fmt.Println(theme.Subtle.Render("nothing staged")) + } else { + for _, s := range pending { + fmt.Println(theme.Subtle.Render(" " + filepath.Base(s.Path))) + } + } + case slash.AttachClear: + n := len(pending) + pending = nil + fmt.Println(theme.Subtle.Render(fmt.Sprintf("cleared %d staged attachment(s)", n))) + } + continue default: // slash.KindSend fmt.Println() spin.start(theme.Status(theme.VerbThinking, 0)) - _, err = session.SendMessage(action.Text) - spin.stop() + files, overrides := attachstage.BuildSend(pending, action) + if len(files) == 0 { + _, err = session.SendMessage(action.Text) + spin.stop() + } else { + var blocked []attachments.Blocked + _, blocked, err = session.SendWithAttachments(ctx, action.Text, files, overrides) + spin.stop() + for _, b := range blocked { + fmt.Fprintln(os.Stderr, theme.Error.Render(theme.Cross+" "+filepath.Base(b.Path)+" — "+b.Reason)) + } + if err == nil { + pending = nil // clear on success only + } + } if err != nil { fmt.Fprintln(os.Stderr, theme.Error.Render(theme.Cross+" "+err.Error())) } diff --git a/extraction/doc.go b/extraction/doc.go new file mode 100644 index 0000000..48e3931 --- /dev/null +++ b/extraction/doc.go @@ -0,0 +1,111 @@ +package extraction + +import ( + "errors" + "fmt" + "image" + "log" + "mime" + "path/filepath" + "strings" +) + +var ( + // ErrNotSupported is returned when an operation (e.g. Image) is not + // available for a given format. + ErrNotSupported = errors.New("extraction: operation not supported for this format") + + // ErrUnsupportedFormat is returned by Open when the file extension is + // not recognised. + ErrUnsupportedFormat = errors.New("extraction: unsupported file format") +) + +// Document provides page-oriented access to text and rendered images. +type Document interface { + // NumPage returns the number of logical pages (sheets, slides, chapters). + NumPage() int + + // Text returns the text content of page pageIndex (0-based). + Text(pageIndex int) (string, error) + + // Image renders page pageIndex as an image. + // Returns ErrNotSupported for formats that cannot render pages. + Image(pageIndex int) (image.Image, error) + + // Close releases all resources held by the document. + Close() error +} + +// documentExtensions lists extensions supported by Open. +var documentExtensions = map[string]bool{ + ".pdf": true, ".epub": true, ".docx": true, ".xlsx": true, ".pptx": true, +} + +// IsDocumentFile returns true for document formats supported by the extraction package. +func IsDocumentFile(path string) bool { + return documentExtensions[strings.ToLower(filepath.Ext(path))] +} + +// IsPDFFile returns true if the file is a PDF. +func IsPDFFile(path string) bool { + return strings.ToLower(filepath.Ext(path)) == ".pdf" +} + +// ExtractText extracts all text from a document, concatenating every page. +// The result is sanitised with strings.ToValidUTF8. +func ExtractText(path string) (string, error) { + doc, err := Open(path) + if err != nil { + return "", fmt.Errorf("extraction open failed: %w", err) + } + defer func() { _ = doc.Close() }() + + var buf strings.Builder + for i := range doc.NumPage() { + text, err := doc.Text(i) + if err != nil { + log.Printf("Warning: text extraction failed for page %d: %v", i, err) + continue + } + buf.WriteString(text) + } + return strings.ToValidUTF8(buf.String(), " "), nil +} + +// mimeToExt maps document MIME types to file extensions. +var mimeToExt = map[string]string{ + "application/pdf": ".pdf", + "application/epub+zip": ".epub", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": ".xlsx", + "application/vnd.openxmlformats-officedocument.presentationml.presentation": ".pptx", +} + +// IsDocumentContentType checks whether the given Content-Type header value +// corresponds to a supported document format. It returns the file extension +// and true if so, or ("", false) otherwise. +func IsDocumentContentType(contentType string) (ext string, ok bool) { + mediaType, _, _ := mime.ParseMediaType(contentType) + ext, ok = mimeToExt[mediaType] + return +} + +// Open opens a document at path and returns a Document whose concrete type +// is chosen by file extension. +func Open(path string) (Document, error) { + ext := strings.ToLower(filepath.Ext(path)) + switch ext { + case ".pdf": + return openPDF(path) + case ".docx": + return openDOCX(path) + case ".xlsx": + return openXLSX(path) + case ".pptx": + return openPPTX(path) + case ".epub": + return openEPUB(path) + default: + return nil, ErrUnsupportedFormat + } +} diff --git a/extraction/doc_test.go b/extraction/doc_test.go new file mode 100644 index 0000000..ffac821 --- /dev/null +++ b/extraction/doc_test.go @@ -0,0 +1,27 @@ +package extraction + +import ( + "strings" + "testing" +) + +func TestExtractTextPDF(t *testing.T) { + got, err := ExtractText("testdata/sample.pdf") + if err != nil { + t.Fatalf("ExtractText: %v", err) + } + // "PDF Form Example" is a stable substring present in the real sample.pdf + // fixture ported from notetaker. + if !strings.Contains(strings.ToLower(got), "pdf form example") { + t.Fatalf("extracted text missing marker; got %q", got) + } +} + +func TestIsDocumentFile(t *testing.T) { + if !IsDocumentFile("a.pdf") || !IsDocumentFile("b.docx") { + t.Fatal("expected pdf/docx to be document files") + } + if IsDocumentFile("c.png") { + t.Fatal("png must not be a document file") + } +} diff --git a/extraction/docx.go b/extraction/docx.go new file mode 100644 index 0000000..7210351 --- /dev/null +++ b/extraction/docx.go @@ -0,0 +1,84 @@ +package extraction + +import ( + "archive/zip" + "encoding/xml" + "image" + "io" + "strings" +) + +type docxDocument struct { + text string +} + +func openDOCX(path string) (*docxDocument, error) { + zr, err := zip.OpenReader(path) + if err != nil { + return nil, err + } + defer zr.Close() + + for _, f := range zr.File { + if f.Name == "word/document.xml" { + rc, err := f.Open() + if err != nil { + return nil, err + } + text, err := parseDOCXBody(rc) + _ = rc.Close() + if err != nil { + return nil, err + } + return &docxDocument{text: text}, nil + } + } + return &docxDocument{}, nil +} + +// parseDOCXBody extracts text from the word/document.xml stream. +func parseDOCXBody(r io.Reader) (string, error) { + dec := xml.NewDecoder(r) + var buf strings.Builder + var inText bool + + for { + tok, err := dec.Token() + if err == io.EOF { + break + } + if err != nil { + return buf.String(), nil // return what we have + } + + switch t := tok.(type) { + case xml.StartElement: + local := t.Name.Local + if local == "t" { + inText = true + } + case xml.EndElement: + local := t.Name.Local + switch local { + case "t": + inText = false + case "p": + buf.WriteByte('\n') + case "br": + buf.WriteByte('\n') + case "tab": + buf.WriteByte('\t') + } + case xml.CharData: + if inText { + buf.Write(t) + } + } + } + return buf.String(), nil +} + +func (d *docxDocument) NumPage() int { return 1 } +func (d *docxDocument) Text(_ int) (string, error) { return d.text, nil } +func (d *docxDocument) Image(_ int) (image.Image, error) { return nil, ErrNotSupported } +func (d *docxDocument) Close() error { return nil } diff --git a/extraction/epub.go b/extraction/epub.go new file mode 100644 index 0000000..97a893e --- /dev/null +++ b/extraction/epub.go @@ -0,0 +1,201 @@ +package extraction + +import ( + "archive/zip" + "encoding/xml" + "image" + "io" + "path" + "strings" +) + +type epubDocument struct { + chapters []string +} + +func openEPUB(filePath string) (*epubDocument, error) { + zr, err := zip.OpenReader(filePath) + if err != nil { + return nil, err + } + defer zr.Close() + + files := zipFiles(zr) + + // 1. Parse container.xml → OPF path + opfPath := findOPFPath(files) + if opfPath == "" { + return &epubDocument{}, nil + } + + opfDir := path.Dir(opfPath) + + // 2. Parse OPF → manifest + spine + manifest, spine := parseOPF(files, opfPath) + + // 3. Extract text from each spine item + var chapters []string + for _, idref := range spine { + href, ok := manifest[idref] + if !ok { + continue + } + // Resolve href relative to OPF directory + var itemPath string + if opfDir == "." { + itemPath = href + } else { + itemPath = opfDir + "/" + href + } + itemPath = path.Clean(itemPath) + + f, ok := files[itemPath] + if !ok { + continue + } + rc, err := f.Open() + if err != nil { + continue + } + text := parseXHTMLText(rc) + _ = rc.Close() + if text != "" { + chapters = append(chapters, text) + } + } + + return &epubDocument{chapters: chapters}, nil +} + +func findOPFPath(files map[string]*zip.File) string { + f, ok := files["META-INF/container.xml"] + if !ok { + return "" + } + rc, err := f.Open() + if err != nil { + return "" + } + defer rc.Close() + + dec := xml.NewDecoder(rc) + for { + tok, err := dec.Token() + if err != nil { + break + } + if se, ok := tok.(xml.StartElement); ok && se.Name.Local == "rootfile" { + for _, a := range se.Attr { + if a.Name.Local == "full-path" { + return a.Value + } + } + } + } + return "" +} + +func parseOPF(files map[string]*zip.File, opfPath string) (manifest map[string]string, spine []string) { + manifest = make(map[string]string) + f, ok := files[opfPath] + if !ok { + return + } + rc, err := f.Open() + if err != nil { + return + } + defer rc.Close() + + dec := xml.NewDecoder(rc) + for { + tok, err := dec.Token() + if err != nil { + break + } + if se, ok := tok.(xml.StartElement); ok { + switch se.Name.Local { + case "item": + var id, href string + for _, a := range se.Attr { + switch a.Name.Local { + case "id": + id = a.Value + case "href": + href = a.Value + } + } + if id != "" && href != "" { + manifest[id] = href + } + case "itemref": + for _, a := range se.Attr { + if a.Name.Local == "idref" { + spine = append(spine, a.Value) + } + } + } + } + } + return +} + +// parseXHTMLText extracts visible text from an XHTML document. +func parseXHTMLText(r io.ReadCloser) string { + dec := xml.NewDecoder(r) + // Tolerate HTML entities that aren't declared in the XML + dec.Strict = false + dec.AutoClose = xml.HTMLAutoClose + dec.Entity = xml.HTMLEntity + + var buf strings.Builder + depth := 0 // track nesting so we can detect block boundaries + + blockElements := map[string]bool{ + "p": true, "div": true, "br": true, "li": true, + "h1": true, "h2": true, "h3": true, "h4": true, "h5": true, "h6": true, + "tr": true, "blockquote": true, "section": true, "article": true, + } + + for { + tok, err := dec.Token() + if err != nil { + break + } + switch t := tok.(type) { + case xml.StartElement: + depth++ + if blockElements[t.Name.Local] && buf.Len() > 0 { + buf.WriteByte('\n') + } + case xml.EndElement: + depth-- + if blockElements[t.Name.Local] && buf.Len() > 0 { + buf.WriteByte('\n') + } + case xml.CharData: + text := strings.TrimSpace(string(t)) + if text != "" { + if buf.Len() > 0 { + last := buf.String()[buf.Len()-1] + if last != '\n' && last != ' ' { + buf.WriteByte(' ') + } + } + buf.WriteString(text) + } + } + } + _ = depth + return strings.TrimSpace(buf.String()) +} + +func (d *epubDocument) NumPage() int { return len(d.chapters) } +func (d *epubDocument) Text(i int) (string, error) { + if i < 0 || i >= len(d.chapters) { + return "", nil + } + return d.chapters[i], nil +} +func (d *epubDocument) Image(_ int) (image.Image, error) { return nil, ErrNotSupported } +func (d *epubDocument) Close() error { return nil } diff --git a/extraction/pdf.go b/extraction/pdf.go new file mode 100644 index 0000000..44a8715 --- /dev/null +++ b/extraction/pdf.go @@ -0,0 +1,130 @@ +package extraction + +import ( + "fmt" + "image" + "os" + "sync" + "time" + + "github.com/klippa-app/go-pdfium" + "github.com/klippa-app/go-pdfium/references" + "github.com/klippa-app/go-pdfium/requests" + "github.com/klippa-app/go-pdfium/webassembly" +) + +var ( + pdfiumPool pdfium.Pool + poolOnce sync.Once + poolErr error +) + +func initPool() { + poolOnce.Do(func() { + pdfiumPool, poolErr = webassembly.Init(webassembly.Config{ + MinIdle: 1, + MaxIdle: 1, + MaxTotal: 4, + }) + }) +} + +type pdfDocument struct { + instance pdfium.Pdfium + doc references.FPDF_DOCUMENT + pages int +} + +func openPDF(path string) (*pdfDocument, error) { + initPool() + if poolErr != nil { + return nil, fmt.Errorf("pdfium init: %w", poolErr) + } + + instance, err := pdfiumPool.GetInstance(30 * time.Second) + if err != nil { + return nil, fmt.Errorf("pdfium instance: %w", err) + } + + // #nosec G304 -- path is internally derived from the staged upload path + data, err := os.ReadFile(path) + if err != nil { + _ = instance.Close() + return nil, err + } + + resp, err := instance.OpenDocument(&requests.OpenDocument{ + File: &data, + }) + if err != nil { + _ = instance.Close() + return nil, fmt.Errorf("pdfium open: %w", err) + } + + pageCount, err := instance.FPDF_GetPageCount(&requests.FPDF_GetPageCount{ + Document: resp.Document, + }) + if err != nil { + _, _ = instance.FPDF_CloseDocument(&requests.FPDF_CloseDocument{Document: resp.Document}) + _ = instance.Close() + return nil, fmt.Errorf("pdfium page count: %w", err) + } + + return &pdfDocument{ + instance: instance, + doc: resp.Document, + pages: pageCount.PageCount, + }, nil +} + +func (d *pdfDocument) NumPage() int { return d.pages } + +func (d *pdfDocument) Text(pageIndex int) (string, error) { + resp, err := d.instance.GetPageText(&requests.GetPageText{ + Page: requests.Page{ + ByIndex: &requests.PageByIndex{ + Document: d.doc, + Index: pageIndex, + }, + }, + }) + if err != nil { + return "", err + } + return resp.Text, nil +} + +func (d *pdfDocument) Image(pageIndex int) (image.Image, error) { + resp, err := d.instance.RenderPageInDPI(&requests.RenderPageInDPI{ + DPI: 150, + Page: requests.Page{ + ByIndex: &requests.PageByIndex{ + Document: d.doc, + Index: pageIndex, + }, + }, + }) + if err != nil { + return nil, err + } + if resp.Result.Image == nil { + if resp.CleanupFunc != nil { + resp.CleanupFunc() + } + return nil, fmt.Errorf("pdfium: nil image for page %d", pageIndex) + } + // Copy pixel data before cleanup frees the WASM-backed buffer. + src := resp.Result.Image + dst := image.NewRGBA(src.Rect) + copy(dst.Pix, src.Pix) + if resp.CleanupFunc != nil { + resp.CleanupFunc() + } + return dst, nil +} + +func (d *pdfDocument) Close() error { + _, _ = d.instance.FPDF_CloseDocument(&requests.FPDF_CloseDocument{Document: d.doc}) + _ = d.instance.Close() + return nil +} diff --git a/extraction/pptx.go b/extraction/pptx.go new file mode 100644 index 0000000..6c1e629 --- /dev/null +++ b/extraction/pptx.go @@ -0,0 +1,123 @@ +package extraction + +import ( + "archive/zip" + "encoding/xml" + "fmt" + "image" + "io" + "strings" +) + +type pptxDocument struct { + slides []string +} + +func openPPTX(filePath string) (*pptxDocument, error) { + zr, err := zip.OpenReader(filePath) + if err != nil { + return nil, err + } + defer zr.Close() + + files := zipFiles(zr) + + // Get slide order from presentation.xml → rels + var slideRIDs []string + if rc := zipOpenPath(files, "ppt/presentation.xml"); rc != nil { + dec := xml.NewDecoder(rc) + for { + tok, err := dec.Token() + if err != nil { + break + } + if se, ok := tok.(xml.StartElement); ok && se.Name.Local == "sldId" { + for _, a := range se.Attr { + // The r:id attribute resolves to a namespace ending in /relationships + if a.Name.Local == "id" && a.Name.Space != "" && + strings.HasSuffix(a.Name.Space, "/relationships") { + slideRIDs = append(slideRIDs, a.Value) + } + } + } + } + _ = rc.Close() + } + + relsMap := parseOOXMLRels(files, "ppt/_rels/presentation.xml.rels") + + var slides []string + if len(slideRIDs) > 0 { + for _, rid := range slideRIDs { + target, ok := relsMap[rid] + if !ok { + slides = append(slides, "") + continue + } + slidePath := "ppt/" + target + rc := zipOpenPath(files, slidePath) + if rc == nil { + slides = append(slides, "") + continue + } + text := parseSlideXML(rc) + _ = rc.Close() + slides = append(slides, text) + } + } else { + // Fallback: read slides sequentially by name + for i := 1; i <= 999; i++ { + slidePath := fmt.Sprintf("ppt/slides/slide%d.xml", i) + rc := zipOpenPath(files, slidePath) + if rc == nil { + break + } + text := parseSlideXML(rc) + _ = rc.Close() + slides = append(slides, text) + } + } + + return &pptxDocument{slides: slides}, nil +} + +func parseSlideXML(r io.ReadCloser) string { + dec := xml.NewDecoder(r) + var buf strings.Builder + var inText bool + + for { + tok, err := dec.Token() + if err != nil { + break + } + switch t := tok.(type) { + case xml.StartElement: + if t.Name.Local == "t" { + inText = true + } + case xml.EndElement: + switch t.Name.Local { + case "t": + inText = false + case "p": + buf.WriteByte('\n') + } + case xml.CharData: + if inText { + buf.Write(t) + } + } + } + return buf.String() +} + +func (d *pptxDocument) NumPage() int { return len(d.slides) } +func (d *pptxDocument) Text(i int) (string, error) { + if i < 0 || i >= len(d.slides) { + return "", nil + } + return d.slides[i], nil +} +func (d *pptxDocument) Image(_ int) (image.Image, error) { return nil, ErrNotSupported } +func (d *pptxDocument) Close() error { return nil } diff --git a/extraction/testdata/sample.docx b/extraction/testdata/sample.docx new file mode 100644 index 0000000..1bac61d Binary files /dev/null and b/extraction/testdata/sample.docx differ diff --git a/extraction/testdata/sample.epub b/extraction/testdata/sample.epub new file mode 100644 index 0000000..ba84a64 Binary files /dev/null and b/extraction/testdata/sample.epub differ diff --git a/extraction/testdata/sample.pdf b/extraction/testdata/sample.pdf new file mode 100644 index 0000000..72d0d21 Binary files /dev/null and b/extraction/testdata/sample.pdf differ diff --git a/extraction/testdata/sample.pptx b/extraction/testdata/sample.pptx new file mode 100644 index 0000000..3f2db57 Binary files /dev/null and b/extraction/testdata/sample.pptx differ diff --git a/extraction/testdata/sample.xlsx b/extraction/testdata/sample.xlsx new file mode 100644 index 0000000..ca336fa Binary files /dev/null and b/extraction/testdata/sample.xlsx differ diff --git a/extraction/text.go b/extraction/text.go new file mode 100644 index 0000000..56f06fe --- /dev/null +++ b/extraction/text.go @@ -0,0 +1,29 @@ +package extraction + +import ( + "os" + "path/filepath" + "strings" +) + +var plainTextExts = map[string]bool{ + ".txt": true, ".md": true, ".markdown": true, ".csv": true, ".json": true, + ".yaml": true, ".yml": true, ".xml": true, ".html": true, ".htm": true, + ".rtf": true, ".log": true, ".toml": true, ".ini": true, ".rst": true, + ".tex": true, ".cfg": true, ".conf": true, +} + +// IsPlainTextFile reports whether path has a known plain-text extension. +func IsPlainTextFile(path string) bool { + return plainTextExts[strings.ToLower(filepath.Ext(path))] +} + +// ReadTextFile reads a plain-text file and returns its content coerced to valid +// UTF-8. HTML/RTF are returned verbatim (no tag stripping) — a documented v1 limit. +func ReadTextFile(path string) (string, error) { + b, err := os.ReadFile(path) + if err != nil { + return "", err + } + return strings.ToValidUTF8(string(b), ""), nil +} diff --git a/extraction/text_test.go b/extraction/text_test.go new file mode 100644 index 0000000..fe3a3fd --- /dev/null +++ b/extraction/text_test.go @@ -0,0 +1,31 @@ +package extraction + +import ( + "os" + "path/filepath" + "testing" +) + +func TestReadTextFile(t *testing.T) { + dir := t.TempDir() + p := filepath.Join(dir, "note.md") + if err := os.WriteFile(p, []byte("# Title\n\ncafé ☕\n"), 0o644); err != nil { + t.Fatal(err) + } + got, err := ReadTextFile(p) + if err != nil { + t.Fatalf("ReadTextFile: %v", err) + } + if got != "# Title\n\ncafé ☕\n" { + t.Fatalf("unexpected content %q", got) + } +} + +func TestIsPlainTextFile(t *testing.T) { + if !IsPlainTextFile("a.md") || !IsPlainTextFile("b.CSV") { + t.Fatal("expected md/CSV to be plain text") + } + if IsPlainTextFile("c.pdf") { + t.Fatal("pdf is not plain text") + } +} diff --git a/extraction/xlsx.go b/extraction/xlsx.go new file mode 100644 index 0000000..6824034 --- /dev/null +++ b/extraction/xlsx.go @@ -0,0 +1,211 @@ +package extraction + +import ( + "archive/zip" + "encoding/xml" + "image" + "io" + "strconv" + "strings" +) + +type xlsxDocument struct { + sheets []string // one text entry per sheet +} + +func openXLSX(filePath string) (*xlsxDocument, error) { + zr, err := zip.OpenReader(filePath) + if err != nil { + return nil, err + } + defer zr.Close() + + files := zipFiles(zr) + + // 1. shared strings + shared := parseSharedStrings(files) + + // 2. workbook → sheet names & rIds + type sheetRef struct { + name string + rID string + } + var sheetRefs []sheetRef + if rc := zipOpenPath(files, "xl/workbook.xml"); rc != nil { + dec := xml.NewDecoder(rc) + for { + tok, err := dec.Token() + if err != nil { + break + } + if se, ok := tok.(xml.StartElement); ok && se.Name.Local == "sheet" { + var name, rid string + for _, a := range se.Attr { + switch a.Name.Local { + case "name": + name = a.Value + case "id": + rid = a.Value + } + } + if rid != "" { + sheetRefs = append(sheetRefs, sheetRef{name, rid}) + } + } + } + _ = rc.Close() + } + + // 3. rels → rId to target path + relsMap := parseOOXMLRels(files, "xl/_rels/workbook.xml.rels") + + // 4. parse each sheet + var sheets []string + for _, sr := range sheetRefs { + target, ok := relsMap[sr.rID] + if !ok { + sheets = append(sheets, "") + continue + } + sheetPath := "xl/" + target + rc := zipOpenPath(files, sheetPath) + if rc == nil { + sheets = append(sheets, "") + continue + } + text := parseSheet(rc, shared) + _ = rc.Close() + sheets = append(sheets, text) + } + + return &xlsxDocument{sheets: sheets}, nil +} + +func parseSharedStrings(files map[string]*zip.File) []string { + rc := zipOpenPath(files, "xl/sharedStrings.xml") + if rc == nil { + return nil + } + defer rc.Close() + + var ss []string + dec := xml.NewDecoder(rc) + var inSI, inT bool + var cur strings.Builder + + for { + tok, err := dec.Token() + if err != nil { + break + } + switch t := tok.(type) { + case xml.StartElement: + switch t.Name.Local { + case "si": + inSI = true + cur.Reset() + case "t": + if inSI { + inT = true + } + } + case xml.EndElement: + switch t.Name.Local { + case "si": + ss = append(ss, cur.String()) + inSI = false + case "t": + inT = false + } + case xml.CharData: + if inT { + cur.Write(t) + } + } + } + return ss +} + +func parseSheet(r io.ReadCloser, shared []string) string { + dec := xml.NewDecoder(r) + var buf strings.Builder + var cellType string + var inV, inIS, inT bool + var cellVal strings.Builder + firstRow := true + firstCell := true + + for { + tok, err := dec.Token() + if err != nil { + break + } + switch t := tok.(type) { + case xml.StartElement: + switch t.Name.Local { + case "row": + if !firstRow { + buf.WriteByte('\n') + } + firstRow = false + firstCell = true + case "c": + cellType = "" + cellVal.Reset() + for _, a := range t.Attr { + if a.Name.Local == "t" { + cellType = a.Value + } + } + case "v": + inV = true + case "is": + inIS = true + case "t": + if inIS { + inT = true + } + } + case xml.EndElement: + switch t.Name.Local { + case "c": + if !firstCell { + buf.WriteByte('\t') + } + firstCell = false + val := cellVal.String() + if cellType == "s" { + idx, err := strconv.Atoi(val) + if err == nil && idx >= 0 && idx < len(shared) { + buf.WriteString(shared[idx]) + } + } else { + buf.WriteString(val) + } + case "v": + inV = false + case "is": + inIS = false + case "t": + inT = false + } + case xml.CharData: + if inV { + cellVal.Write(t) + } else if inT && inIS { + cellVal.Write(t) + } + } + } + return buf.String() +} + +func (d *xlsxDocument) NumPage() int { return len(d.sheets) } +func (d *xlsxDocument) Text(i int) (string, error) { + if i < 0 || i >= len(d.sheets) { + return "", nil + } + return d.sheets[i], nil +} +func (d *xlsxDocument) Image(_ int) (image.Image, error) { return nil, ErrNotSupported } +func (d *xlsxDocument) Close() error { return nil } diff --git a/extraction/zip_helpers.go b/extraction/zip_helpers.go new file mode 100644 index 0000000..f4ce488 --- /dev/null +++ b/extraction/zip_helpers.go @@ -0,0 +1,64 @@ +package extraction + +import ( + "archive/zip" + "encoding/xml" + "io" + "path" +) + +// zipFiles builds a name→File map for quick lookups. +func zipFiles(zr *zip.ReadCloser) map[string]*zip.File { + m := make(map[string]*zip.File, len(zr.File)) + for _, f := range zr.File { + m[f.Name] = f + } + return m +} + +// zipOpenPath opens a file in the zip by cleaned path name. +func zipOpenPath(files map[string]*zip.File, name string) io.ReadCloser { + name = path.Clean(name) + f, ok := files[name] + if !ok { + return nil + } + rc, err := f.Open() + if err != nil { + return nil + } + return rc +} + +// parseOOXMLRels parses an OOXML .rels file and returns a map of Id→Target. +func parseOOXMLRels(files map[string]*zip.File, relsPath string) map[string]string { + m := make(map[string]string) + rc := zipOpenPath(files, relsPath) + if rc == nil { + return m + } + defer rc.Close() + + dec := xml.NewDecoder(rc) + for { + tok, err := dec.Token() + if err != nil { + break + } + if se, ok := tok.(xml.StartElement); ok && se.Name.Local == "Relationship" { + var id, target string + for _, a := range se.Attr { + switch a.Name.Local { + case "Id": + id = a.Value + case "Target": + target = a.Value + } + } + if id != "" { + m[id] = target + } + } + } + return m +} diff --git a/go.mod b/go.mod index f0a7c4e..c34e1c6 100644 --- a/go.mod +++ b/go.mod @@ -11,8 +11,9 @@ require ( github.com/charmbracelet/bubbletea v1.3.10 github.com/charmbracelet/glamour v1.0.0 github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834 + github.com/klippa-app/go-pdfium v1.12.2 github.com/modelcontextprotocol/go-sdk v1.0.0 - github.com/mudler/cogito v0.11.0 + github.com/mudler/cogito v0.11.1-0.20260705102758-a15985a850c6 github.com/mudler/xlog v0.0.1 github.com/sashabaranov/go-openai v1.41.2 golang.org/x/net v0.43.0 @@ -38,6 +39,7 @@ require ( github.com/google/uuid v1.6.0 // indirect github.com/gorilla/css v1.0.1 // indirect github.com/huandu/xstrings v1.5.0 // indirect + github.com/jolestar/go-commons-pool/v2 v2.1.2 // indirect github.com/lucasb-eyer/go-colorful v1.3.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-localereader v0.0.1 // indirect @@ -52,6 +54,7 @@ require ( github.com/rivo/uniseg v0.4.7 // indirect github.com/shopspring/decimal v1.4.0 // indirect github.com/spf13/cast v1.7.0 // indirect + github.com/tetratelabs/wazero v1.7.3 // indirect github.com/tmc/langchaingo v0.1.13 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect diff --git a/go.sum b/go.sum index 24cd108..a3a7b5f 100644 --- a/go.sum +++ b/go.sum @@ -59,6 +59,7 @@ github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpS github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= @@ -77,6 +78,8 @@ github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6 github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= +github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= @@ -103,8 +106,12 @@ github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUq github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI= github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= +github.com/jolestar/go-commons-pool/v2 v2.1.2 h1:E+XGo58F23t7HtZiC/W6jzO2Ux2IccSH/yx4nD+J1CM= +github.com/jolestar/go-commons-pool/v2 v2.1.2/go.mod h1:r4NYccrkS5UqP1YQI1COyTZ9UjPJAAGTUxzcsK1kqhY= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klippa-app/go-pdfium v1.12.2 h1:0z9/njA0XwHbzicCHmRoGW32yeTwOfRPpGuxcZN2Arg= +github.com/klippa-app/go-pdfium v1.12.2/go.mod h1:Vw30mehpmosf+bOWjTAPi/ALhO4B5aBO7xZYC0fKH9c= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -146,8 +153,8 @@ github.com/modelcontextprotocol/go-sdk v1.0.0 h1:Z4MSjLi38bTgLrd/LjSmofqRqyBiVKR github.com/modelcontextprotocol/go-sdk v1.0.0/go.mod h1:nYtYQroQ2KQiM0/SbyEPUWQ6xs4B95gJjEalc9AQyOs= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= -github.com/mudler/cogito v0.11.0 h1:of/o3/w402H9vm6mE+RktJVHJpUeOH5kSxSnwgR5Hd8= -github.com/mudler/cogito v0.11.0/go.mod h1:6sfja3lcu2nWRzEc0wwqGNu/eCG3EWgij+8s7xyUeQ4= +github.com/mudler/cogito v0.11.1-0.20260705102758-a15985a850c6 h1:gAXhCNCbUsFTGnZrmAvFqVqosdUGCgwPV7rk7cHZ2yE= +github.com/mudler/cogito v0.11.1-0.20260705102758-a15985a850c6/go.mod h1:6sfja3lcu2nWRzEc0wwqGNu/eCG3EWgij+8s7xyUeQ4= github.com/mudler/xlog v0.0.1 h1:yR3/wszd3ZM6u1n96YITJZ4yUcDgqHSwvQmzUJa+8vg= github.com/mudler/xlog v0.0.1/go.mod h1:39f5vcd05Qd6GWKM8IjyHNQ7AmOx3ZM0YfhfIGhC18U= github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI= @@ -188,10 +195,14 @@ github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/testcontainers/testcontainers-go v0.38.0 h1:d7uEapLcv2P8AvH8ahLqDMMxda2W9gQN1nRbHS28HBw= github.com/testcontainers/testcontainers-go v0.38.0/go.mod h1:C52c9MoHpWO+C4aqmgSU+hxlR5jlEayWtgYrb8Pzz1w= +github.com/tetratelabs/wazero v1.7.3 h1:PBH5KVahrt3S2AHgEjKu4u+LlDbbk+nsGE3KLucy6Rw= +github.com/tetratelabs/wazero v1.7.3/go.mod h1:ytl6Zuh20R/eROuyDaGPkp82O9C/DJfXAwJfQ3X6/7Y= github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= @@ -240,5 +251,6 @@ golang.org/x/tools v0.37.0 h1:DVSRzp7FwePZW356yEAChSdNcQo6Nsp+fex1SUW09lE= golang.org/x/tools v0.37.0/go.mod h1:MBN5QPQtLMHVdvsbtarmTNukZDdgwdwlO5qGacAzF0w= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/slash/attach_test.go b/slash/attach_test.go new file mode 100644 index 0000000..6f323c3 --- /dev/null +++ b/slash/attach_test.go @@ -0,0 +1,43 @@ +package slash + +import ( + "os" + "path/filepath" + "testing" +) + +func TestParseAtPaths(t *testing.T) { + dir := t.TempDir() + real := filepath.Join(dir, "report.pdf") + _ = os.WriteFile(real, []byte("x"), 0o644) + + text, files := parseAtPaths("summarize @" + real + " for @someone please") + if len(files) != 1 || files[0] != real { + t.Fatalf("expected the real file attached, got %v", files) + } + if text != "summarize for @someone please" && text != "summarize for @someone please" { + t.Fatalf("real @path stripped, literal @someone kept; got %q", text) + } +} + +func TestResolveAttach(t *testing.T) { + dir := t.TempDir() + f := filepath.Join(dir, "a.wav") + _ = os.WriteFile(f, []byte("x"), 0o644) + + if a := Resolve("/attach "+f, nil, nil, nil); a.Kind != KindAttach || a.AttachOp != AttachStage || a.AttachPath != f || a.Transcribe { + t.Fatalf("plain stage: %+v", a) + } + if a := Resolve("/attach -t "+f, nil, nil, nil); a.AttachOp != AttachStage || !a.Transcribe { + t.Fatalf("-t stage: %+v", a) + } + if a := Resolve("/attach", nil, nil, nil); a.AttachOp != AttachList { + t.Fatalf("list: %+v", a) + } + if a := Resolve("/attach clear", nil, nil, nil); a.AttachOp != AttachClear { + t.Fatalf("clear: %+v", a) + } + if a := Resolve("/attach /no/such/file", nil, nil, nil); a.Kind != KindError { + t.Fatalf("missing file must be KindError: %+v", a) + } +} diff --git a/slash/slash.go b/slash/slash.go index 3fe7a91..6e917f2 100644 --- a/slash/slash.go +++ b/slash/slash.go @@ -28,6 +28,16 @@ const ( KindGoalSet // set/replace the session goal (Text) KindGoalShow // show the current goal KindGoalClear // clear the current goal + KindAttach // stage/list/clear file attachments +) + +// AttachOp enumerates the /attach sub-operations. +type AttachOp int + +const ( + AttachStage AttachOp = iota // stage AttachPath (optionally Transcribe) + AttachList // list staged attachments + AttachClear // clear staged attachments ) // Action is the resolved result of a submitted input line. @@ -41,6 +51,12 @@ type Action struct { Interval time.Duration // KindLoopStart: 0 = self-paced Payload string // KindLoopStart: the prompt/slash-command to repeat LoopID string // KindLoopStop: empty = stop all + + // Attachment actions: + Files []string // KindSend: resolved @path attachments + AttachOp AttachOp // KindAttach: which op + AttachPath string // KindAttach+AttachStage: file to stage + Transcribe bool // KindAttach+AttachStage: --transcribe/-t override } // Expand renders a command's prompt template with the given args. @@ -64,7 +80,8 @@ func Expand(c types.CommandConfig, args string) (string, error) { func Resolve(input string, cmds []types.CommandConfig, skills []types.Skill, agents []types.AgentTypeConfig) Action { trimmed := strings.TrimSpace(input) if !strings.HasPrefix(trimmed, "/") { - return Action{Kind: KindSend, Text: input} + text, files := parseAtPaths(input) + return Action{Kind: KindSend, Text: text, Files: files} } verb, rest := splitVerb(trimmed[1:]) @@ -94,6 +111,25 @@ func Resolve(input string, cmds []types.CommandConfig, skills []types.Skill, age return resolveLoop(rest) case "goal": return resolveGoal(rest) + case "attach": + rest = strings.TrimSpace(rest) + switch { + case rest == "": + return Action{Kind: KindAttach, AttachOp: AttachList} + case rest == "clear": + return Action{Kind: KindAttach, AttachOp: AttachClear} + default: + transcribe := false + if f, ok := strings.CutPrefix(rest, "--transcribe "); ok { + transcribe, rest = true, strings.TrimSpace(f) + } else if f, ok := strings.CutPrefix(rest, "-t "); ok { + transcribe, rest = true, strings.TrimSpace(f) + } + if _, err := os.Stat(rest); err != nil { + return Action{Kind: KindError, Err: "no such file: " + rest} + } + return Action{Kind: KindAttach, AttachOp: AttachStage, AttachPath: rest, Transcribe: transcribe} + } default: c, ok := findCommand(cmds, verb) if !ok { @@ -154,6 +190,49 @@ func resolveGoal(rest string) Action { return Action{Kind: KindGoalSet, Text: rest} } +// parseAtPaths splits a send line into literal text and @path attachments. A +// @token is attached only if it resolves to an existing file (cwd-relative or +// absolute); @"quoted paths" are supported; unmatched @tokens stay literal. +func parseAtPaths(input string) (string, []string) { + var files []string + var out strings.Builder + i := 0 + for i < len(input) { + atBoundary := i == 0 || input[i-1] == ' ' || input[i-1] == '\t' || input[i-1] == '\n' + if input[i] == '@' && atBoundary && i+1 < len(input) { + tok, next := readToken(input, i+1) + if tok != "" { + if _, err := os.Stat(tok); err == nil { + files = append(files, tok) + i = next + continue // drop the token from the text + } + } + } + out.WriteByte(input[i]) + i++ + } + return strings.TrimSpace(out.String()), files +} + +// readToken reads a path token starting at j: a "quoted string" (up to the +// closing quote) or an unquoted run up to whitespace. Returns the token and the +// index just past it. +func readToken(s string, j int) (string, int) { + if j < len(s) && s[j] == '"' { + k := strings.IndexByte(s[j+1:], '"') + if k >= 0 { + return s[j+1 : j+1+k], j + 1 + k + 1 + } + return "", j + } + k := j + for k < len(s) && s[k] != ' ' && s[k] != '\t' && s[k] != '\n' { + k++ + } + return s[j:k], k +} + // delegation builds a directive instructing the agent to delegate to a named // sub-agent (the runtime already exposes spawn_agent + the agent-type list). func delegation(agent, task string) string { diff --git a/slash/slash_test.go b/slash/slash_test.go index bc36bd3..f168145 100644 --- a/slash/slash_test.go +++ b/slash/slash_test.go @@ -1,6 +1,7 @@ package slash import ( + "reflect" "strings" "testing" "time" @@ -121,7 +122,7 @@ func TestResolveGoal(t *testing.T) { } for _, c := range cases { got := Resolve(c.in, nil, nil, nil) - if got != c.want { + if !reflect.DeepEqual(got, c.want) { t.Errorf("Resolve(%q) = %+v, want %+v", c.in, got, c.want) } } diff --git a/specialist/client.go b/specialist/client.go new file mode 100644 index 0000000..c0703ac --- /dev/null +++ b/specialist/client.go @@ -0,0 +1,15 @@ +package specialist + +import "net/http" + +// Client calls LocalAI's OpenAI-compatible specialist endpoints (transcription, +// vision description). baseURL is the LocalAI base, e.g. http://localhost:8080/v1. +type Client struct { + baseURL string + apiKey string + http *http.Client +} + +func New(baseURL, apiKey string) *Client { + return &Client{baseURL: baseURL, apiKey: apiKey, http: http.DefaultClient} +} diff --git a/specialist/describe.go b/specialist/describe.go new file mode 100644 index 0000000..406ca46 --- /dev/null +++ b/specialist/describe.go @@ -0,0 +1,91 @@ +package specialist + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "mime" + "net/http" + "os" + "path/filepath" +) + +// DataURI reads a file and returns a base64 data: URI (e.g. data:image/png;base64,...). +// Exported so the attachments package reuses it for native image parts. +func DataURI(path string) (string, error) { + b, err := os.ReadFile(path) + if err != nil { + return "", err + } + ct := mime.TypeByExtension(filepath.Ext(path)) + if ct == "" { + ct = "application/octet-stream" + } + return "data:" + ct + ";base64," + base64.StdEncoding.EncodeToString(b), nil +} + +// Describe turns an image into text by sending it to a vision model and +// returning the model's textual description. +// +// It is the building block for the future Phase 2 `read_image` tool, which will +// let any agent (including a text-only one) pull an image into the conversation +// as model-generated text on demand. It is deliberately NOT wired into the +// attachment routing path: that path Blocks images on text-only models and nudges +// the user to switch to a vision model, rather than silently substituting a lossy +// AI-generated description for the actual image. (Audio, by contrast, auto-falls +// back to faithful transcription — see attachments.Route.) So Describe is +// intentionally ahead of its consumer in this branch. +func (c *Client) Describe(ctx context.Context, path, model, prompt string) (string, error) { + if prompt == "" { + prompt = "Describe this image in detail." + } + uri, err := DataURI(path) + if err != nil { + return "", err + } + payload := map[string]any{ + "model": model, // empty → LocalAI auto-selects a vision model + "messages": []map[string]any{{ + "role": "user", + "content": []map[string]any{ + {"type": "text", "text": prompt}, + {"type": "image_url", "image_url": map[string]string{"url": uri}}, + }, + }}, + } + buf, _ := json.Marshal(payload) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+"/chat/completions", bytes.NewReader(buf)) + if err != nil { + return "", err + } + req.Header.Set("Content-Type", "application/json") + if c.apiKey != "" { + req.Header.Set("Authorization", "Bearer "+c.apiKey) + } + resp, err := c.http.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + b, _ := io.ReadAll(resp.Body) + return "", fmt.Errorf("describe failed: %s: %s", resp.Status, b) + } + var out struct { + Choices []struct { + Message struct { + Content string `json:"content"` + } `json:"message"` + } `json:"choices"` + } + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return "", err + } + if len(out.Choices) == 0 { + return "", fmt.Errorf("describe: empty choices") + } + return out.Choices[0].Message.Content, nil +} diff --git a/specialist/describe_test.go b/specialist/describe_test.go new file mode 100644 index 0000000..549c64f --- /dev/null +++ b/specialist/describe_test.go @@ -0,0 +1,54 @@ +package specialist + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestDescribe(t *testing.T) { + var sawImagePart bool + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req struct { + Messages []struct { + Content []struct { + Type string `json:"type"` + ImageURL *struct { + URL string `json:"url"` + } `json:"image_url"` + } `json:"content"` + } `json:"messages"` + } + _ = json.NewDecoder(r.Body).Decode(&req) + for _, m := range req.Messages { + for _, part := range m.Content { + if part.Type == "image_url" && part.ImageURL != nil && strings.HasPrefix(part.ImageURL.URL, "data:image/") { + sawImagePart = true + } + } + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"choices":[{"message":{"content":"a red square"}}]}`)) + })) + defer srv.Close() + + dir := t.TempDir() + img := filepath.Join(dir, "x.png") + _ = os.WriteFile(img, []byte("\x89PNG\r\n\x1a\nfakepng"), 0o644) + + got, err := New(srv.URL, "").Describe(context.Background(), img, "", "") + if err != nil { + t.Fatalf("Describe: %v", err) + } + if !strings.Contains(got, "red square") { + t.Fatalf("unexpected description %q", got) + } + if !sawImagePart { + t.Fatal("expected a data:image/ image_url part") + } +} diff --git a/specialist/transcribe.go b/specialist/transcribe.go new file mode 100644 index 0000000..8ed6046 --- /dev/null +++ b/specialist/transcribe.go @@ -0,0 +1,64 @@ +package specialist + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "mime/multipart" + "net/http" + "os" + "path/filepath" +) + +// Transcribe uploads the audio file to {baseURL}/audio/transcriptions and returns +// the text. An empty model lets LocalAI pick the first FLAG_TRANSCRIPT model. +func (c *Client) Transcribe(ctx context.Context, path, model string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + + var body bytes.Buffer + mw := multipart.NewWriter(&body) + if model != "" { + _ = mw.WriteField("model", model) + } + part, err := mw.CreateFormFile("file", filepath.Base(path)) + if err != nil { + return "", err + } + if _, err := io.Copy(part, f); err != nil { + return "", err + } + if err := mw.Close(); err != nil { + return "", err + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+"/audio/transcriptions", &body) + if err != nil { + return "", err + } + req.Header.Set("Content-Type", mw.FormDataContentType()) + if c.apiKey != "" { + req.Header.Set("Authorization", "Bearer "+c.apiKey) + } + resp, err := c.http.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + b, _ := io.ReadAll(resp.Body) + return "", fmt.Errorf("transcription failed: %s: %s", resp.Status, b) + } + var out struct { + Text string `json:"text"` + } + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return "", err + } + return out.Text, nil +} diff --git a/specialist/transcribe_test.go b/specialist/transcribe_test.go new file mode 100644 index 0000000..014e752 --- /dev/null +++ b/specialist/transcribe_test.go @@ -0,0 +1,46 @@ +package specialist + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestTranscribe(t *testing.T) { + var gotModel, gotFile string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := r.ParseMultipartForm(1 << 20); err != nil { + t.Fatalf("parse multipart: %v", err) + } + gotModel = r.FormValue("model") + if f, _, err := r.FormFile("file"); err == nil { + gotFile = "present" + _ = f + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"text":"hello from parakeet"}`)) + })) + defer srv.Close() + + dir := t.TempDir() + audio := filepath.Join(dir, "clip.wav") + _ = os.WriteFile(audio, []byte("RIFFxxxxWAVE"), 0o644) + + got, err := New(srv.URL, "").Transcribe(context.Background(), audio, "") + if err != nil { + t.Fatalf("Transcribe: %v", err) + } + if !strings.Contains(got, "hello from parakeet") { + t.Fatalf("unexpected transcript %q", got) + } + if gotFile != "present" { + t.Fatal("expected multipart file field") + } + if gotModel != "" { + t.Fatalf("expected empty model (auto-select), got %q", gotModel) + } +} diff --git a/tui/model.go b/tui/model.go index c4d759d..18af463 100644 --- a/tui/model.go +++ b/tui/model.go @@ -19,6 +19,8 @@ import ( "github.com/charmbracelet/glamour" "github.com/charmbracelet/lipgloss" "github.com/modelcontextprotocol/go-sdk/mcp" + "github.com/mudler/nib/attachments" + "github.com/mudler/nib/attachstage" "github.com/mudler/nib/chat" "github.com/mudler/nib/loop" wizmcp "github.com/mudler/nib/mcp" @@ -141,6 +143,11 @@ type Model struct { queue []string queueSel int + // pending holds files staged via /attach, awaiting the next message. They + // combine with inline @path files (via attachstage.BuildSend) on send and + // clear only after a successful send (mirrors the CLI REPL). + pending []attachstage.StagedFile + // redispatch holds follow-ups that were released into a run (and echoed) // but never consumed by it — the run ended first. They re-dispatch as // fresh turns ahead of the queue, without a second echo, and are not @@ -164,6 +171,7 @@ type Model struct { type responseMsg struct { content string err error + blocked []attachments.Blocked } // compactResultMsg is the outcome of a manual /compact run. @@ -698,6 +706,16 @@ func (m Model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.interruptArmed = false m.status = "" m.reasoning = "" + // Surface any attachments that couldn't be sent (blocked by model caps + // or resolution), mirroring the CLI's per-file error lines. + for _, b := range msg.blocked { + m.messages = append(m.messages, ChatMessage{Role: "error", Content: filepath.Base(b.Path) + " — " + b.Reason}) + } + // Clear staged attachments only on a successful send (retain on error), + // matching the CLI REPL. + if msg.err == nil { + m.pending = nil + } if msg.err != nil { if errors.Is(msg.err, context.Canceled) { m.messages = append(m.messages, ChatMessage{Role: "agent", Content: "interrupted."}) @@ -1055,11 +1073,43 @@ func (m *Model) dispatchResolved(input string) tea.Cmd { m.messages = append(m.messages, ChatMessage{Role: "agent", Content: "No goal to clear."}) } return nil + case slash.KindAttach: + switch action.AttachOp { + case slash.AttachStage: + m.pending = append(m.pending, attachstage.StagedFile{Path: action.AttachPath, Transcribe: action.Transcribe}) + mode := "default" + if action.Transcribe { + mode = "transcribe" + } + m.messages = append(m.messages, ChatMessage{Role: "agent", Content: "attached: " + filepath.Base(action.AttachPath) + " (" + mode + ") — sends with your next message"}) + case slash.AttachList: + if len(m.pending) == 0 { + m.messages = append(m.messages, ChatMessage{Role: "agent", Content: "nothing staged"}) + } else { + var b strings.Builder + for i, s := range m.pending { + if i > 0 { + b.WriteString("\n") + } + b.WriteString(filepath.Base(s.Path)) + } + m.messages = append(m.messages, ChatMessage{Role: "agent", Content: b.String()}) + } + case slash.AttachClear: + n := len(m.pending) + m.pending = nil + m.messages = append(m.messages, ChatMessage{Role: "agent", Content: fmt.Sprintf("cleared %d staged attachment(s)", n)}) + } + return nil default: // slash.KindSend + files, overrides := attachstage.BuildSend(m.pending, action) m.loading = true m.interruptArmed = false m.status = "" - return m.sendMessage(action.Text) + if len(files) == 0 { + return m.sendMessage(action.Text) + } + return m.sendWithAttachmentsCmd(action.Text, files, overrides) } } @@ -1071,6 +1121,15 @@ func (m Model) sendMessage(text string) tea.Cmd { } } +// sendWithAttachmentsCmd sends a message with staged + inline @path attachments. +// Blocked entries and clear-on-success are handled in the responseMsg handler. +func (m Model) sendWithAttachmentsCmd(text string, files []string, overrides map[string]attachments.Override) tea.Cmd { + return func() tea.Msg { + reply, blocked, err := m.session.SendWithAttachments(m.ctx, text, files, overrides) + return responseMsg{content: reply, err: err, blocked: blocked} + } +} + // shellTickMsg drives a periodic refresh of the shell-jobs footer. type shellTickMsg struct{} diff --git a/tui/queue.go b/tui/queue.go index 3effe66..8dc5a79 100644 --- a/tui/queue.go +++ b/tui/queue.go @@ -57,6 +57,12 @@ func (m *Model) releaseQueueFront() bool { if action.Kind != slash.KindSend { return false } + // InjectUser only carries text; it can't convey @path attachments. Leaving an + // attachment-bearing entry queued lets flushQueueAsTurn → dispatchInput handle + // it at end-of-run, where attachments are honored — so the files aren't dropped. + if len(action.Files) > 0 { + return false + } // InjectUser (not Inject) so a follow-up the run never consumes is handed // back at run end (TakeUndelivered) and re-dispatched instead of lost. if !m.session.InjectUser(action.Text) { diff --git a/types/config.go b/types/config.go index b6e927e..5e6968b 100644 --- a/types/config.go +++ b/types/config.go @@ -85,11 +85,15 @@ type HookConfig struct { // Config holds configuration for creating a new session type Config struct { - Model string `yaml:"model"` - APIKey string `yaml:"api_key"` - BaseURL string `yaml:"base_url"` - LogLevel string `yaml:"log_level"` - Prompt string `yaml:"prompt"` + Model string `yaml:"model"` + APIKey string `yaml:"api_key"` + BaseURL string `yaml:"base_url"` + // Specialist models for attachment handling. Empty ⇒ LocalAI auto-selects + // by usecase (FLAG_TRANSCRIPT / FLAG_VISION). + TranscribeModel string `yaml:"transcribe_model,omitempty" json:"transcribe_model,omitempty"` + VisionModel string `yaml:"vision_model,omitempty" json:"vision_model,omitempty"` + LogLevel string `yaml:"log_level"` + Prompt string `yaml:"prompt"` // Metadata is a per-request metadata object attached verbatim to every // chat-completion request (the OpenAI "metadata" field). Backends such as // LocalAI use it for per-request flags, e.g. {"enable_thinking": "false"}