Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 10 additions & 3 deletions lib/Service/OpenAiAPIService.php
Original file line number Diff line number Diff line change
Expand Up @@ -451,6 +451,7 @@ public function createStreamedChatCompletion(
?string $toolMessage = null,
?array $tools = null,
?array $files = null,
bool $includeFileIds = false,
): \Generator {
if ($this->isQuotaExceeded($userId, Application::QUOTA_TYPE_TEXT, $service)) {
throw new Exception($this->l10n->t('Text generation quota exceeded'), Http::STATUS_TOO_MANY_REQUESTS);
Expand All @@ -470,6 +471,7 @@ public function createStreamedChatCompletion(
$tools,
$files,
true,
$includeFileIds,
);

$response = $this->request(
Expand Down Expand Up @@ -511,11 +513,12 @@ public function createChatCompletion(
?string $toolMessage = null,
?array $tools = null,
?array $files = null,
bool $includeFileIds = false,
): array {
$response = $this->requestChatCompletion(
$userId, $service, $model, $userPrompt, $systemPrompt, $history,
$n, $maxTokens, $extraParams, $toolMessage, $tools, $files,
false,
false, $includeFileIds,
);

if (isset($response['usage'], $response['usage']['total_tokens'])) {
Expand Down Expand Up @@ -560,6 +563,7 @@ public function requestChatCompletion(
?array $tools = null,
?array $files = null,
bool $stream = false,
bool $includeFileIds = false,
): array {
if ($this->isQuotaExceeded($userId, Application::QUOTA_TYPE_TEXT, $service)) {
throw new Exception($this->l10n->t('Text generation quota exceeded'), Http::STATUS_TOO_MANY_REQUESTS);
Expand All @@ -579,6 +583,7 @@ public function requestChatCompletion(
$tools,
$files,
$stream,
$includeFileIds,
);

return $this->request($userId, $service, 'chat/completions', $params, 'POST');
Expand All @@ -597,6 +602,7 @@ public function requestChatCompletion(
* @param array|null $tools
* @param array|null $files Array of File objects
* @param bool $stream
* @param bool $includeFileIds Expose the IDs of attached files to the model so it can pass them to tools
* @return array<string, mixed>
*/
private function buildChatCompletionRequestParams(
Expand All @@ -613,6 +619,7 @@ private function buildChatCompletionRequestParams(
?array $tools = null,
?array $files = null,
bool $stream = false,
bool $includeFileIds = false,
): array {
$modelRequestParam = $this->modelParam($service, $model, Application::DEFAULT_COMPLETION_MODEL_ID);

Expand Down Expand Up @@ -673,7 +680,7 @@ private function buildChatCompletionRequestParams(
}
// If the history contains a file that isn't supported anymore we should skip it so the chat isn't broken
try {
$content = array_merge($content, $this->openAiFileService->buildFileContentFromId($item['file_id'], $userId, $item['ocp_task_id'] ?? null, $service));
$content = array_merge($content, $this->openAiFileService->buildFileContentFromId($item['file_id'], $userId, $item['ocp_task_id'] ?? null, $service, $includeFileIds));
} catch (ProcessingException|UserFacingProcessingException $e) {
$this->logger->warning('Could not build file content from id: ' . $item['file_id'] . '. Error: ' . $e->getMessage(), ['app' => Application::APP_ID]);
}
Expand All @@ -693,7 +700,7 @@ private function buildChatCompletionRequestParams(
}
$content = [];
foreach ($files as $file) {
$content = array_merge($content, $this->openAiFileService->buildFileContentFromFile($file, $service, $userId));
$content = array_merge($content, $this->openAiFileService->buildFileContentFromFile($file, $service, $userId, $includeFileIds));
}
if ($userPrompt !== null) {
$content[] = [
Expand Down
39 changes: 36 additions & 3 deletions lib/Service/OpenAiFileService.php
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
use OCA\OpenAi\Vendor\Smalot\PdfParser\Parser;
use OCP\Files\File;
use OCP\Files\IRootFolder;
use OCP\Files\NotPermittedException;
use OCP\IL10N;
use OCP\TaskProcessing\Exception\ProcessingException;
use OCP\TaskProcessing\Exception\UserFacingProcessingException;
Expand Down Expand Up @@ -86,11 +87,12 @@ public function __construct(
* @param ?string $userId The user ID.
* @param ?int $taskId The ID of the task
* @param ServiceConfig $service The service the content is built for
* @param bool $includeFileId Whether to prepend a text part exposing the file ID to the model (e.g. for tool calls)
* @return list<array<string, mixed>> Content parts suitable for OpenAI chat message content.
* @throws ProcessingException
* @throws UserFacingProcessingException
*/
public function buildFileContentFromId(int $fileId, ?string $userId, ?int $taskId, ServiceConfig $service): array {
public function buildFileContentFromId(int $fileId, ?string $userId, ?int $taskId, ServiceConfig $service, bool $includeFileId = false): array {
$file = null;
if ($taskId !== null) {
$task = $this->taskProcessingManager->getUserTask($taskId, $userId);
Expand All @@ -109,22 +111,53 @@ public function buildFileContentFromId(int $fileId, ?string $userId, ?int $taskI
$userFolder = $this->rootFolder->getUserFolder($userId);
$file = $userFolder->getFirstNodeById($fileId);
}
return $this->buildFileContentFromFile($file, $service, $userId);
return $this->buildFileContentFromFile($file, $service, $userId, $includeFileId);
}

/**
* Builds file content from a File object.
*
* @param ?File $file The file to build content from.
* @param ServiceConfig $service The service the content is built for
* @param bool $includeFileId Whether to prepend a text part exposing the file ID to the model (e.g. for tool calls)
* @return list<array<string, mixed>> Content parts suitable for OpenAI chat message content.
* @throws ProcessingException
* @throws UserFacingProcessingException
*/
public function buildFileContentFromFile(?File $file, ServiceConfig $service, ?string $userId = null): array {
public function buildFileContentFromFile(?File $file, ServiceConfig $service, ?string $userId = null, bool $includeFileId = false): array {
if (!$file instanceof File || !$file->isReadable()) {
throw new ProcessingException('File is not readable');
}
$content = $this->buildContentForFileType($file, $service, $userId);
if ($includeFileId && $this->isInUserFolder($file, $userId)) {
array_unshift($content, [
'type' => 'text',
'text' => 'Attached file "' . $file->getName() . '" (file ID: ' . $file->getId() . ')',
]);
}
return $content;
}

/**
* Only files in the user's folder can be used by tools, files in appdata (e.g. task outputs) cannot
*/
private function isInUserFolder(File $file, ?string $userId): bool {
if ($userId === null) {
return false;
}
try {
return $this->rootFolder->getUserFolder($userId)->getFirstNodeById($file->getId()) !== null;
} catch (NotPermittedException $e) {
return false;
}
}
Comment on lines +141 to +153

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: maybe it's enough to check the file path here if it's in appdata or not


/**
* @return list<array<string, mixed>>
* @throws ProcessingException
* @throws UserFacingProcessingException
*/
private function buildContentForFileType(File $file, ServiceConfig $service, ?string $userId): array {
// Maximum file size for openai is 50MB.
if ($service->isUsingOpenAi() && $file->getSize() > self::MAX_FILE_SIZE_BYTES) {
throw new UserFacingProcessingException(
Expand Down
4 changes: 2 additions & 2 deletions lib/TaskProcessing/MultimodalChatWithToolsProvider.php
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@ public function process(
try {
if ($preferStreaming) {
$chunks = $this->openAiAPIService->createStreamedChatCompletion(
$userId, $this->service, $this->model, $userPrompt, $systemPrompt, $history, 1, $maxTokens, null, $toolMessage, $tools, $inputAttachments
$userId, $this->service, $this->model, $userPrompt, $systemPrompt, $history, 1, $maxTokens, null, $toolMessage, $tools, $inputAttachments, true
);
$time = microtime(true);
$streamedOutput = '';
Expand Down Expand Up @@ -197,7 +197,7 @@ public function process(
$returnValue = $chunks->getReturn();
} else {
$returnValue = $this->openAiAPIService->createChatCompletion(
$userId, $this->service, $this->model, $userPrompt, $systemPrompt, $history, 1, $maxTokens, null, $toolMessage, $tools, $inputAttachments
$userId, $this->service, $this->model, $userPrompt, $systemPrompt, $history, 1, $maxTokens, null, $toolMessage, $tools, $inputAttachments, true
);
}
} catch (UserFacingProcessingException $e) {
Expand Down
17 changes: 16 additions & 1 deletion tests/unit/Providers/OpenAiProviderTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,10 @@ class OpenAiProviderTest extends TestCase {
* @var MockObject|IClient
*/
private $iClient;
/**
* @var MockObject|\OCP\Files\IRootFolder
*/
private $rootFolder;
private QuotaUsageMapper $quotaUsageMapper;

public static function setUpBeforeClass(): void {
Expand Down Expand Up @@ -112,6 +116,8 @@ protected function setUp(): void {
'multimodal_image_enabled' => true,
]);

$this->rootFolder = $this->createMock(\OCP\Files\IRootFolder::class);

$this->openAiApiService = new OpenAiAPIService(
\OCP\Server::get(\Psr\Log\LoggerInterface::class),
$this->createMock(\OCP\IL10N::class),
Expand All @@ -122,7 +128,7 @@ protected function setUp(): void {
$this->streamingService,
new OpenAiFileService(
$this->createMock(\OCP\IL10N::class),
$this->createMock(\OCP\Files\IRootFolder::class),
$this->rootFolder,
$this->createMock(\OCP\TaskProcessing\IManager::class),
$this->createMock(\Psr\Log\LoggerInterface::class),
),
Expand Down Expand Up @@ -1405,6 +1411,11 @@ public function testMultimodalChatWithToolsProvider(): void {
$file->method('getSize')->willReturn(strlen($imageContent));
$file->method('getMimeType')->willReturn('image/jpeg');
$file->method('fopen')->with('rb')->willReturn($stream);
$file->method('getName')->willReturn('photo.jpg');
$file->method('getId')->willReturn(42);
$userFolder = $this->createMock(\OCP\Files\Folder::class);
$userFolder->method('getFirstNodeById')->with(42)->willReturn($file);
$this->rootFolder->method('getUserFolder')->with(self::TEST_USER1)->willReturn($userFolder);

$response = '{
"id": "chatcmpl-123",
Expand Down Expand Up @@ -1440,6 +1451,10 @@ public function testMultimodalChatWithToolsProvider(): void {
[
'role' => 'user',
'content' => [
[
'type' => 'text',
'text' => 'Attached file "photo.jpg" (file ID: 42)',
],
[
'type' => 'image_url',
'image_url' => [
Expand Down
Loading