diff --git a/lib/Service/OpenAiAPIService.php b/lib/Service/OpenAiAPIService.php index 56d121b0..55bd8fe6 100644 --- a/lib/Service/OpenAiAPIService.php +++ b/lib/Service/OpenAiAPIService.php @@ -451,6 +451,7 @@ public function createStreamedChatCompletion( ?string $toolMessage = null, ?array $tools = null, ?array $files = null, + bool $includeFileIds = false, ): \Generator { if ($this->isQuotaExceeded($userId, Application::QUOTA_TYPE_TEXT, $service)) { throw new Exception($this->l10n->t('Text generation quota exceeded'), Http::STATUS_TOO_MANY_REQUESTS); @@ -470,6 +471,7 @@ public function createStreamedChatCompletion( $tools, $files, true, + $includeFileIds, ); $response = $this->request( @@ -511,11 +513,12 @@ public function createChatCompletion( ?string $toolMessage = null, ?array $tools = null, ?array $files = null, + bool $includeFileIds = false, ): array { $response = $this->requestChatCompletion( $userId, $service, $model, $userPrompt, $systemPrompt, $history, $n, $maxTokens, $extraParams, $toolMessage, $tools, $files, - false, + false, $includeFileIds, ); if (isset($response['usage'], $response['usage']['total_tokens'])) { @@ -560,6 +563,7 @@ public function requestChatCompletion( ?array $tools = null, ?array $files = null, bool $stream = false, + bool $includeFileIds = false, ): array { if ($this->isQuotaExceeded($userId, Application::QUOTA_TYPE_TEXT, $service)) { throw new Exception($this->l10n->t('Text generation quota exceeded'), Http::STATUS_TOO_MANY_REQUESTS); @@ -579,6 +583,7 @@ public function requestChatCompletion( $tools, $files, $stream, + $includeFileIds, ); return $this->request($userId, $service, 'chat/completions', $params, 'POST'); @@ -597,6 +602,7 @@ public function requestChatCompletion( * @param array|null $tools * @param array|null $files Array of File objects * @param bool $stream + * @param bool $includeFileIds Expose the IDs of attached files to the model so it can pass them to tools * @return array */ private function buildChatCompletionRequestParams( @@ -613,6 +619,7 @@ private function buildChatCompletionRequestParams( ?array $tools = null, ?array $files = null, bool $stream = false, + bool $includeFileIds = false, ): array { $modelRequestParam = $this->modelParam($service, $model, Application::DEFAULT_COMPLETION_MODEL_ID); @@ -673,7 +680,7 @@ private function buildChatCompletionRequestParams( } // If the history contains a file that isn't supported anymore we should skip it so the chat isn't broken try { - $content = array_merge($content, $this->openAiFileService->buildFileContentFromId($item['file_id'], $userId, $item['ocp_task_id'] ?? null, $service)); + $content = array_merge($content, $this->openAiFileService->buildFileContentFromId($item['file_id'], $userId, $item['ocp_task_id'] ?? null, $service, $includeFileIds)); } catch (ProcessingException|UserFacingProcessingException $e) { $this->logger->warning('Could not build file content from id: ' . $item['file_id'] . '. Error: ' . $e->getMessage(), ['app' => Application::APP_ID]); } @@ -693,7 +700,7 @@ private function buildChatCompletionRequestParams( } $content = []; foreach ($files as $file) { - $content = array_merge($content, $this->openAiFileService->buildFileContentFromFile($file, $service, $userId)); + $content = array_merge($content, $this->openAiFileService->buildFileContentFromFile($file, $service, $userId, $includeFileIds)); } if ($userPrompt !== null) { $content[] = [ diff --git a/lib/Service/OpenAiFileService.php b/lib/Service/OpenAiFileService.php index b4856b58..80b77db6 100644 --- a/lib/Service/OpenAiFileService.php +++ b/lib/Service/OpenAiFileService.php @@ -14,6 +14,7 @@ use OCA\OpenAi\Vendor\Smalot\PdfParser\Parser; use OCP\Files\File; use OCP\Files\IRootFolder; +use OCP\Files\NotPermittedException; use OCP\IL10N; use OCP\TaskProcessing\Exception\ProcessingException; use OCP\TaskProcessing\Exception\UserFacingProcessingException; @@ -86,11 +87,12 @@ public function __construct( * @param ?string $userId The user ID. * @param ?int $taskId The ID of the task * @param ServiceConfig $service The service the content is built for + * @param bool $includeFileId Whether to prepend a text part exposing the file ID to the model (e.g. for tool calls) * @return list> Content parts suitable for OpenAI chat message content. * @throws ProcessingException * @throws UserFacingProcessingException */ - public function buildFileContentFromId(int $fileId, ?string $userId, ?int $taskId, ServiceConfig $service): array { + public function buildFileContentFromId(int $fileId, ?string $userId, ?int $taskId, ServiceConfig $service, bool $includeFileId = false): array { $file = null; if ($taskId !== null) { $task = $this->taskProcessingManager->getUserTask($taskId, $userId); @@ -109,7 +111,7 @@ public function buildFileContentFromId(int $fileId, ?string $userId, ?int $taskI $userFolder = $this->rootFolder->getUserFolder($userId); $file = $userFolder->getFirstNodeById($fileId); } - return $this->buildFileContentFromFile($file, $service, $userId); + return $this->buildFileContentFromFile($file, $service, $userId, $includeFileId); } /** @@ -117,14 +119,45 @@ public function buildFileContentFromId(int $fileId, ?string $userId, ?int $taskI * * @param ?File $file The file to build content from. * @param ServiceConfig $service The service the content is built for + * @param bool $includeFileId Whether to prepend a text part exposing the file ID to the model (e.g. for tool calls) * @return list> Content parts suitable for OpenAI chat message content. * @throws ProcessingException * @throws UserFacingProcessingException */ - public function buildFileContentFromFile(?File $file, ServiceConfig $service, ?string $userId = null): array { + public function buildFileContentFromFile(?File $file, ServiceConfig $service, ?string $userId = null, bool $includeFileId = false): array { if (!$file instanceof File || !$file->isReadable()) { throw new ProcessingException('File is not readable'); } + $content = $this->buildContentForFileType($file, $service, $userId); + if ($includeFileId && $this->isInUserFolder($file, $userId)) { + array_unshift($content, [ + 'type' => 'text', + 'text' => 'Attached file "' . $file->getName() . '" (file ID: ' . $file->getId() . ')', + ]); + } + return $content; + } + + /** + * Only files in the user's folder can be used by tools, files in appdata (e.g. task outputs) cannot + */ + private function isInUserFolder(File $file, ?string $userId): bool { + if ($userId === null) { + return false; + } + try { + return $this->rootFolder->getUserFolder($userId)->getFirstNodeById($file->getId()) !== null; + } catch (NotPermittedException $e) { + return false; + } + } + + /** + * @return list> + * @throws ProcessingException + * @throws UserFacingProcessingException + */ + private function buildContentForFileType(File $file, ServiceConfig $service, ?string $userId): array { // Maximum file size for openai is 50MB. if ($service->isUsingOpenAi() && $file->getSize() > self::MAX_FILE_SIZE_BYTES) { throw new UserFacingProcessingException( diff --git a/lib/TaskProcessing/MultimodalChatWithToolsProvider.php b/lib/TaskProcessing/MultimodalChatWithToolsProvider.php index f460bdd2..582be7cc 100644 --- a/lib/TaskProcessing/MultimodalChatWithToolsProvider.php +++ b/lib/TaskProcessing/MultimodalChatWithToolsProvider.php @@ -159,7 +159,7 @@ public function process( try { if ($preferStreaming) { $chunks = $this->openAiAPIService->createStreamedChatCompletion( - $userId, $this->service, $this->model, $userPrompt, $systemPrompt, $history, 1, $maxTokens, null, $toolMessage, $tools, $inputAttachments + $userId, $this->service, $this->model, $userPrompt, $systemPrompt, $history, 1, $maxTokens, null, $toolMessage, $tools, $inputAttachments, true ); $time = microtime(true); $streamedOutput = ''; @@ -197,7 +197,7 @@ public function process( $returnValue = $chunks->getReturn(); } else { $returnValue = $this->openAiAPIService->createChatCompletion( - $userId, $this->service, $this->model, $userPrompt, $systemPrompt, $history, 1, $maxTokens, null, $toolMessage, $tools, $inputAttachments + $userId, $this->service, $this->model, $userPrompt, $systemPrompt, $history, 1, $maxTokens, null, $toolMessage, $tools, $inputAttachments, true ); } } catch (UserFacingProcessingException $e) { diff --git a/tests/unit/Providers/OpenAiProviderTest.php b/tests/unit/Providers/OpenAiProviderTest.php index 74df1cbe..09312ae1 100644 --- a/tests/unit/Providers/OpenAiProviderTest.php +++ b/tests/unit/Providers/OpenAiProviderTest.php @@ -74,6 +74,10 @@ class OpenAiProviderTest extends TestCase { * @var MockObject|IClient */ private $iClient; + /** + * @var MockObject|\OCP\Files\IRootFolder + */ + private $rootFolder; private QuotaUsageMapper $quotaUsageMapper; public static function setUpBeforeClass(): void { @@ -112,6 +116,8 @@ protected function setUp(): void { 'multimodal_image_enabled' => true, ]); + $this->rootFolder = $this->createMock(\OCP\Files\IRootFolder::class); + $this->openAiApiService = new OpenAiAPIService( \OCP\Server::get(\Psr\Log\LoggerInterface::class), $this->createMock(\OCP\IL10N::class), @@ -122,7 +128,7 @@ protected function setUp(): void { $this->streamingService, new OpenAiFileService( $this->createMock(\OCP\IL10N::class), - $this->createMock(\OCP\Files\IRootFolder::class), + $this->rootFolder, $this->createMock(\OCP\TaskProcessing\IManager::class), $this->createMock(\Psr\Log\LoggerInterface::class), ), @@ -1405,6 +1411,11 @@ public function testMultimodalChatWithToolsProvider(): void { $file->method('getSize')->willReturn(strlen($imageContent)); $file->method('getMimeType')->willReturn('image/jpeg'); $file->method('fopen')->with('rb')->willReturn($stream); + $file->method('getName')->willReturn('photo.jpg'); + $file->method('getId')->willReturn(42); + $userFolder = $this->createMock(\OCP\Files\Folder::class); + $userFolder->method('getFirstNodeById')->with(42)->willReturn($file); + $this->rootFolder->method('getUserFolder')->with(self::TEST_USER1)->willReturn($userFolder); $response = '{ "id": "chatcmpl-123", @@ -1440,6 +1451,10 @@ public function testMultimodalChatWithToolsProvider(): void { [ 'role' => 'user', 'content' => [ + [ + 'type' => 'text', + 'text' => 'Attached file "photo.jpg" (file ID: 42)', + ], [ 'type' => 'image_url', 'image_url' => [