diff --git a/docs/components/platform.rst b/docs/components/platform.rst index eff1eaf5ec..8a9a58e5a6 100644 --- a/docs/components/platform.rst +++ b/docs/components/platform.rst @@ -461,6 +461,57 @@ Thanks to Symfony's Cache component, platform calls can be cached to reduce call echo $secondResult->getContent().\PHP_EOL; +High availability +----------------- + +As most platform exposes a REST API, errors can occurs during generation phase due to network issues, timeout and more. + +To prevent exceptions at the application level and allows to keep a smooth experience for end users, +the ``Symfony\AI\Platform\FailoverPlatform`` can be used to automatically call a backup platform:: + + use Symfony\AI\Platform\Bridge\Ollama\PlatformFactory as OllamaPlatformFactory; + use Symfony\AI\Platform\Bridge\OpenAi\PlatformFactory as OpenAiPlatformFactory; + use Symfony\AI\Platform\FailoverPlatform; + use Symfony\AI\Platform\Message\Message; + use Symfony\AI\Platform\Message\MessageBag; + + $ollamaPlatform = OllamaPlatformFactory::create('http://127.0.0.1:11434', http_client()); + $openAiPlatform = OpenAiPlatformFactory::create('sk-foo', http_client()); + + $failoverPlatform = new FailoverPlatform([ + $ollamaPlatform, // # Ollama will fail as 'gpt-4o' is not available in the catalog + $openAiPlatform, + ]); + + $result = $failoverPlatform->invoke('gpt-4o', new MessageBag( + Message::forSystem('You are a helpful assistant.'), + Message::ofUser('Tina has one brother and one sister. How many sisters do Tina\'s siblings have?'), + )); + + echo $result->asText().\PHP_EOL; + +This platform can also be configured when using the bundle:: + + ai: + platform: + openai: + # ... + ollama: + # ... + failover: + ollama_to_openai: + platforms: + - 'ai.platform.ollama' + - 'ai.platform.openai' + retry_period: 120 + +A ``retry_period`` can be configured to determine after how many seconds a failed platform must be tried again, +default value is ``60``. + +.. note:: + + Platforms are executed in the order they're injected into ``FailoverPlatform``. + Testing Tools ------------- diff --git a/examples/misc/failover-transport.php b/examples/misc/failover-transport.php new file mode 100644 index 0000000000..f4ed771631 --- /dev/null +++ b/examples/misc/failover-transport.php @@ -0,0 +1,33 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +use Symfony\AI\Platform\Bridge\Ollama\PlatformFactory as OllamaPlatformFactory; +use Symfony\AI\Platform\Bridge\OpenAi\PlatformFactory as OpenAiPlatformFactory; +use Symfony\AI\Platform\FailoverPlatform; +use Symfony\AI\Platform\Message\Message; +use Symfony\AI\Platform\Message\MessageBag; + +require_once dirname(__DIR__).'/bootstrap.php'; + +$ollamaPlatform = OllamaPlatformFactory::create(env('OLLAMA_HOST_URL'), http_client()); +$openAiPlatform = OpenAiPlatformFactory::create(env('OPENAI_API_KEY'), http_client()); + +$platform = new FailoverPlatform([ + $ollamaPlatform, // # Ollama will fail as 'gpt-4o' is not available in the catalog + $openAiPlatform, +]); + +$result = $platform->invoke('gpt-4o', new MessageBag( + Message::forSystem('You are a helpful assistant.'), + Message::ofUser('Tina has one brother and one sister. How many sisters do Tina\'s siblings have?'), +)); + +echo $result->asText().\PHP_EOL; diff --git a/src/ai-bundle/config/options.php b/src/ai-bundle/config/options.php index adc0a656cb..88cc447d35 100644 --- a/src/ai-bundle/config/options.php +++ b/src/ai-bundle/config/options.php @@ -115,6 +115,19 @@ ->end() ->end() ->end() + ->arrayNode('failover') + ->useAttributeAsKey('name') + ->arrayPrototype() + ->children() + ->arrayNode('platforms') + ->scalarPrototype()->end() + ->end() + ->integerNode('retry_period') + ->defaultValue(60) + ->end() + ->end() + ->end() + ->end() ->arrayNode('gemini') ->children() ->stringNode('api_key')->isRequired()->end() diff --git a/src/ai-bundle/src/AiBundle.php b/src/ai-bundle/src/AiBundle.php index 6810c6c031..f42b09055e 100644 --- a/src/ai-bundle/src/AiBundle.php +++ b/src/ai-bundle/src/AiBundle.php @@ -13,6 +13,7 @@ use Google\Auth\ApplicationDefaultCredentials; use Google\Auth\FetchAuthTokenInterface; +use Psr\Log\LoggerInterface; use Symfony\AI\Agent\Agent; use Symfony\AI\Agent\AgentInterface; use Symfony\AI\Agent\Attribute\AsInputProcessor; @@ -76,6 +77,7 @@ use Symfony\AI\Platform\CachedPlatform; use Symfony\AI\Platform\Capability; use Symfony\AI\Platform\Exception\RuntimeException; +use Symfony\AI\Platform\FailoverPlatform; use Symfony\AI\Platform\Message\Content\File; use Symfony\AI\Platform\ModelCatalog\ModelCatalogInterface; use Symfony\AI\Platform\ModelClientInterface; @@ -489,6 +491,29 @@ private function processPlatformConfig(string $type, array $platform, ContainerB return; } + if ('failover' === $type) { + foreach ($platform as $name => $config) { + $definition = (new Definition(FailoverPlatform::class)) + ->setLazy(true) + ->setArguments([ + array_map( + static fn (string $wrappedPlatform): Reference => new Reference($wrappedPlatform), + $config['platforms'], + ), + new Reference(ClockInterface::class), + $config['retry_period'], + new Reference(LoggerInterface::class), + ]) + ->addTag('proxy', ['interface' => PlatformInterface::class]) + ->addTag('ai.platform', ['name' => $type]); + + $container->setDefinition('ai.platform.'.$type.'.'.$name, $definition); + $container->registerAliasForArgument('ai.platform.'.$type.'.'.$name, PlatformInterface::class, $name); + } + + return; + } + if ('gemini' === $type) { $platformId = 'ai.platform.gemini'; $definition = (new Definition(Platform::class)) diff --git a/src/ai-bundle/tests/DependencyInjection/AiBundleTest.php b/src/ai-bundle/tests/DependencyInjection/AiBundleTest.php index c59c9f5f71..ad6022fab3 100644 --- a/src/ai-bundle/tests/DependencyInjection/AiBundleTest.php +++ b/src/ai-bundle/tests/DependencyInjection/AiBundleTest.php @@ -18,6 +18,8 @@ use PHPUnit\Framework\Attributes\TestWith; use PHPUnit\Framework\TestCase; use Probots\Pinecone\Client as PineconeClient; +use Psr\Log\LoggerInterface; +use Psr\Log\NullLogger; use Symfony\AI\Agent\AgentInterface; use Symfony\AI\Agent\Memory\MemoryInputProcessor; use Symfony\AI\Agent\Memory\StaticMemoryProvider; @@ -34,6 +36,7 @@ use Symfony\AI\Platform\Bridge\Ollama\OllamaApiCatalog; use Symfony\AI\Platform\Capability; use Symfony\AI\Platform\EventListener\TemplateRendererListener; +use Symfony\AI\Platform\FailoverPlatform; use Symfony\AI\Platform\Message\TemplateRenderer\ExpressionLanguageTemplateRenderer; use Symfony\AI\Platform\Message\TemplateRenderer\StringTemplateRenderer; use Symfony\AI\Platform\Message\TemplateRenderer\TemplateRendererRegistry; @@ -74,6 +77,8 @@ use Symfony\AI\Store\ManagedStoreInterface; use Symfony\AI\Store\RetrieverInterface; use Symfony\AI\Store\StoreInterface; +use Symfony\Component\Clock\ClockInterface; +use Symfony\Component\Clock\MonotonicClock; use Symfony\Component\Config\Definition\Exception\InvalidConfigurationException; use Symfony\Component\DependencyInjection\ContainerBuilder; use Symfony\Component\DependencyInjection\ContainerInterface; @@ -4016,6 +4021,104 @@ public function testElevenLabsPlatformWithApiCatalogCanBeRegistered() $this->assertSame([['interface' => ModelCatalogInterface::class]], $modelCatalogDefinition->getTag('proxy')); } + public function testFailoverPlatformCanBeCreated() + { + $container = $this->buildContainer([ + 'ai' => [ + 'platform' => [ + 'ollama' => [ + 'host_url' => 'http://127.0.0.1:11434', + ], + 'openai' => [ + 'api_key' => 'sk-openai_key_full', + ], + 'failover' => [ + 'main' => [ + 'platforms' => [ + 'ai.platform.ollama', + 'ai.platform.openai', + ], + ], + ], + ], + ], + ]); + + $this->assertTrue($container->hasDefinition('ai.platform.failover.main')); + + $definition = $container->getDefinition('ai.platform.failover.main'); + + $this->assertTrue($definition->isLazy()); + $this->assertSame(FailoverPlatform::class, $definition->getClass()); + + $this->assertCount(4, $definition->getArguments()); + $this->assertCount(2, $definition->getArgument(0)); + $this->assertEquals([ + new Reference('ai.platform.ollama'), + new Reference('ai.platform.openai'), + ], $definition->getArgument(0)); + $this->assertInstanceOf(Reference::class, $definition->getArgument(1)); + $this->assertSame(ClockInterface::class, (string) $definition->getArgument(1)); + $this->assertSame(60, $definition->getArgument(2)); + $this->assertInstanceOf(Reference::class, $definition->getArgument(3)); + $this->assertSame(LoggerInterface::class, (string) $definition->getArgument(3)); + + $this->assertTrue($definition->hasTag('proxy')); + $this->assertSame([['interface' => PlatformInterface::class]], $definition->getTag('proxy')); + $this->assertTrue($definition->hasTag('ai.platform')); + $this->assertSame([['name' => 'failover']], $definition->getTag('ai.platform')); + + $this->assertTrue($container->hasAlias('Symfony\AI\Platform\PlatformInterface $main')); + + $container = $this->buildContainer([ + 'ai' => [ + 'platform' => [ + 'ollama' => [ + 'host_url' => 'http://127.0.0.1:11434', + ], + 'openai' => [ + 'api_key' => 'sk-openai_key_full', + ], + 'failover' => [ + 'main' => [ + 'platforms' => [ + 'ai.platform.ollama', + 'ai.platform.openai', + ], + 'retry_period' => 120, + ], + ], + ], + ], + ]); + + $this->assertTrue($container->hasDefinition('ai.platform.failover.main')); + + $definition = $container->getDefinition('ai.platform.failover.main'); + + $this->assertTrue($definition->isLazy()); + $this->assertSame(FailoverPlatform::class, $definition->getClass()); + + $this->assertCount(4, $definition->getArguments()); + $this->assertCount(2, $definition->getArgument(0)); + $this->assertEquals([ + new Reference('ai.platform.ollama'), + new Reference('ai.platform.openai'), + ], $definition->getArgument(0)); + $this->assertInstanceOf(Reference::class, $definition->getArgument(1)); + $this->assertSame(ClockInterface::class, (string) $definition->getArgument(1)); + $this->assertSame(120, $definition->getArgument(2)); + $this->assertInstanceOf(Reference::class, $definition->getArgument(3)); + $this->assertSame(LoggerInterface::class, (string) $definition->getArgument(3)); + + $this->assertTrue($definition->hasTag('proxy')); + $this->assertSame([['interface' => PlatformInterface::class]], $definition->getTag('proxy')); + $this->assertTrue($definition->hasTag('ai.platform')); + $this->assertSame([['name' => 'failover']], $definition->getTag('ai.platform')); + + $this->assertTrue($container->hasAlias('Symfony\AI\Platform\PlatformInterface $main')); + } + public function testOpenAiPlatformWithDefaultRegion() { $container = $this->buildContainer([ @@ -6987,6 +7090,8 @@ private function buildContainer(array $configuration): ContainerBuilder $container->setParameter('kernel.debug', true); $container->setParameter('kernel.environment', 'dev'); $container->setParameter('kernel.build_dir', 'public'); + $container->setDefinition(ClockInterface::class, new Definition(MonotonicClock::class)); + $container->setDefinition(LoggerInterface::class, new Definition(NullLogger::class)); $extension = (new AiBundle())->getContainerExtension(); $extension->load($configuration, $container); @@ -7042,6 +7147,21 @@ private function getFullConfig(): array 'host' => 'https://api.elevenlabs.io/v1', 'api_key' => 'elevenlabs_key_full', ], + 'failover' => [ + 'main' => [ + 'platforms' => [ + 'ai.platform.ollama', + 'ai.platform.openai', + ], + ], + 'main_with_custom_retry_period' => [ + 'platforms' => [ + 'ai.platform.ollama', + 'ai.platform.openai', + ], + 'retry_period' => 120, + ], + ], 'gemini' => [ 'api_key' => 'gemini_key_full', ], diff --git a/src/platform/src/Exception/LogicException.php b/src/platform/src/Exception/LogicException.php new file mode 100644 index 0000000000..bad5360ac6 --- /dev/null +++ b/src/platform/src/Exception/LogicException.php @@ -0,0 +1,19 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Symfony\AI\Platform\Exception; + +/** + * @author Guillaume Loulier + */ +final class LogicException extends \LogicException implements ExceptionInterface +{ +} diff --git a/src/platform/src/FailoverPlatform.php b/src/platform/src/FailoverPlatform.php new file mode 100644 index 0000000000..4332b1c868 --- /dev/null +++ b/src/platform/src/FailoverPlatform.php @@ -0,0 +1,87 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Symfony\AI\Platform; + +use Psr\Log\LoggerInterface; +use Psr\Log\NullLogger; +use Symfony\AI\Platform\Exception\LogicException; +use Symfony\AI\Platform\Exception\RuntimeException; +use Symfony\AI\Platform\ModelCatalog\ModelCatalogInterface; +use Symfony\AI\Platform\Result\DeferredResult; +use Symfony\Component\Clock\ClockInterface; +use Symfony\Component\Clock\MonotonicClock; + +/** + * @author Guillaume Loulier + */ +final class FailoverPlatform implements PlatformInterface +{ + /** + * @var \SplObjectStorage + */ + private \SplObjectStorage $deadPlatforms; + + /** + * @param PlatformInterface[] $platforms + */ + public function __construct( + private readonly iterable $platforms, + private readonly ClockInterface $clock = new MonotonicClock(), + private readonly int $retryPeriod = 60, + private readonly LoggerInterface $logger = new NullLogger(), + ) { + if ([] === $platforms) { + throw new LogicException(\sprintf('"%s" must have at least one platform configured.', self::class)); + } + + $this->deadPlatforms = new \SplObjectStorage(); + } + + public function invoke(string $model, object|array|string $input, array $options = []): DeferredResult + { + return $this->do(static fn (PlatformInterface $platform): DeferredResult => $platform->invoke($model, $input, $options)); + } + + public function getModelCatalog(): ModelCatalogInterface + { + return $this->do(static fn (PlatformInterface $platform): ModelCatalogInterface => $platform->getModelCatalog()); + } + + private function do(\Closure $func): DeferredResult|ModelCatalogInterface + { + foreach ($this->platforms as $platform) { + if ($this->deadPlatforms->offsetExists($platform) && ($this->clock->now()->getTimestamp() - $this->deadPlatforms->offsetGet($platform)) > $this->retryPeriod) { + $this->deadPlatforms->offsetUnset($platform); + } + + if ($this->deadPlatforms->offsetExists($platform)) { + continue; + } + + try { + return $func($platform); + } catch (\Throwable $throwable) { + $this->deadPlatforms->offsetSet($platform, $this->clock->now()->getTimestamp()); + + $this->logger->error('The {platform} platform due to an error/exception: {message}', [ + 'platform' => $platform::class, + 'message' => $throwable->getMessage(), + 'exception' => $throwable, + ]); + + continue; + } + } + + throw new RuntimeException('All platforms failed.'); + } +} diff --git a/src/platform/tests/FailoverPlatformTest.php b/src/platform/tests/FailoverPlatformTest.php new file mode 100644 index 0000000000..0a160aa241 --- /dev/null +++ b/src/platform/tests/FailoverPlatformTest.php @@ -0,0 +1,288 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Symfony\AI\Platform\Tests; + +use PHPUnit\Framework\Attributes\Group; +use PHPUnit\Framework\TestCase; +use Psr\Log\LoggerInterface; +use Symfony\AI\Platform\Exception\RuntimeException; +use Symfony\AI\Platform\FailoverPlatform; +use Symfony\AI\Platform\ModelCatalog\FallbackModelCatalog; +use Symfony\AI\Platform\ModelCatalog\ModelCatalogInterface; +use Symfony\AI\Platform\PlatformInterface; +use Symfony\AI\Platform\Result\DeferredResult; +use Symfony\AI\Platform\Result\RawHttpResult; +use Symfony\AI\Platform\Result\TextResult; +use Symfony\AI\Platform\ResultConverterInterface; +use Symfony\AI\Platform\Test\InMemoryPlatform; +use Symfony\Component\Clock\MonotonicClock; +use Symfony\Contracts\HttpClient\ResponseInterface; + +#[Group('time-sensitive')] +final class FailoverPlatformTest extends TestCase +{ + public function testPlatformCannotPerformInvokeWithoutRemainingPlatform() + { + $mainPlatform = $this->createMock(PlatformInterface::class); + $mainPlatform->expects($this->once())->method('invoke') + ->willThrowException(new \Exception()); + + $failedPlatform = $this->createMock(PlatformInterface::class); + $failedPlatform->expects($this->once())->method('invoke') + ->willThrowException(new \Exception()); + + $logger = $this->createMock(LoggerInterface::class); + $logger->expects($this->exactly(2))->method('error'); + + $failoverPlatform = new FailoverPlatform([ + $failedPlatform, + $mainPlatform, + ], logger: $logger); + + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage('All platforms failed.'); + $this->expectExceptionCode(0); + $failoverPlatform->invoke('foo', 'foo'); + } + + public function testPlatformCannotRetrieveModelCatalogWithoutRemainingPlatform() + { + $mainPlatform = $this->createMock(PlatformInterface::class); + $mainPlatform->expects($this->once())->method('getModelCatalog') + ->willThrowException(new \Exception()); + + $failedPlatform = $this->createMock(PlatformInterface::class); + $failedPlatform->expects($this->once())->method('getModelCatalog') + ->willThrowException(new \Exception()); + + $logger = $this->createMock(LoggerInterface::class); + $logger->expects($this->exactly(2))->method('error'); + + $failoverPlatform = new FailoverPlatform([ + $failedPlatform, + $mainPlatform, + ], logger: $logger); + + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage('All platforms failed.'); + $this->expectExceptionCode(0); + $failoverPlatform->getModelCatalog(); + } + + public function testPlatformCanPerformInvokeWithRemainingPlatform() + { + $failedPlatform = $this->createMock(PlatformInterface::class); + $failedPlatform->expects($this->once())->method('invoke') + ->willThrowException(new \Exception()); + + $logger = $this->createMock(LoggerInterface::class); + $logger->expects($this->once())->method('error'); + + $failoverPlatform = new FailoverPlatform([ + $failedPlatform, + new InMemoryPlatform(static fn (): string => 'foo'), + ], logger: $logger); + + $result = $failoverPlatform->invoke('foo', 'foo'); + + $this->assertSame('foo', $result->asText()); + } + + public function testPlatformCanRetrieveModelCatalogWithRemainingPlatform() + { + $failedPlatform = $this->createMock(PlatformInterface::class); + $failedPlatform->expects($this->once())->method('getModelCatalog') + ->willThrowException(new \Exception()); + + $logger = $this->createMock(LoggerInterface::class); + $logger->expects($this->once())->method('error'); + + $failoverPlatform = new FailoverPlatform([ + $failedPlatform, + new InMemoryPlatform(static fn (): string => 'foo'), + ], logger: $logger); + + $this->assertInstanceOf(FallbackModelCatalog::class, $failoverPlatform->getModelCatalog()); + } + + public function testPlatformCanPerformInvokeWhileRemovingPlatformAfterRetryPeriod() + { + $clock = new MonotonicClock(); + + $failedPlatform = $this->createMock(PlatformInterface::class); + $failedPlatform->expects($this->any())->method('invoke') + ->willReturnCallback(function (): DeferredResult { + static $call = 0; + + if (1 === ++$call) { + throw new \Exception('An error occurred'); + } + + $httpResponse = $this->createMock(ResponseInterface::class); + $rawHttpResult = new RawHttpResult($httpResponse); + + $resultConverter = $this->createMock(ResultConverterInterface::class); + $resultConverter->expects($this->once())->method('convert')->willReturn(new TextResult('foo')); + + return new DeferredResult($resultConverter, $rawHttpResult); + }); + + $logger = $this->createMock(LoggerInterface::class); + $logger->expects($this->once())->method('error'); + + $failoverPlatform = new FailoverPlatform([ + $failedPlatform, + new InMemoryPlatform(static fn (): string => 'bar'), + ], clock: $clock, retryPeriod: 1, logger: $logger); + + $firstResult = $failoverPlatform->invoke('foo', 'foo'); + + $this->assertSame('bar', $firstResult->asText()); + + $clock->sleep(4); + + $finalResult = $failedPlatform->invoke('foo', 'bar'); + + $this->assertSame('foo', $finalResult->asText()); + } + + public function testPlatformCanRetrieveModelCatalogWhileRemovingPlatformAfterRetryPeriod() + { + $clock = new MonotonicClock(); + + $failedPlatform = $this->createMock(PlatformInterface::class); + $failedPlatform->expects($this->any())->method('getModelCatalog') + ->willReturnCallback(function (): ModelCatalogInterface { + static $call = 0; + + if (1 === ++$call) { + throw new \Exception('An error occurred'); + } + + return new FallbackModelCatalog(); + }); + + $logger = $this->createMock(LoggerInterface::class); + $logger->expects($this->once())->method('error'); + + $failoverPlatform = new FailoverPlatform([ + $failedPlatform, + new InMemoryPlatform(static fn (): string => 'bar'), + ], clock: $clock, retryPeriod: 1, logger: $logger); + + $this->assertInstanceOf(FallbackModelCatalog::class, $failoverPlatform->getModelCatalog()); + + $clock->sleep(4); + + $this->assertInstanceOf(FallbackModelCatalog::class, $failoverPlatform->getModelCatalog()); + } + + public function testPlatformCannotPerformInvokeWhileAllPlatformFailedDuringRetryPeriod() + { + $clock = new MonotonicClock(); + + $firstPlatform = $this->createMock(PlatformInterface::class); + $firstPlatform->expects($this->any())->method('invoke') + ->willReturnCallback(function (): DeferredResult { + static $call = 0; + + if (1 === ++$call) { + $httpResponse = $this->createMock(ResponseInterface::class); + $rawHttpResult = new RawHttpResult($httpResponse); + + $resultConverter = $this->createMock(ResultConverterInterface::class); + $resultConverter->expects($this->once())->method('convert')->willReturn(new TextResult('foo')); + + return new DeferredResult($resultConverter, $rawHttpResult); + } + + throw new \Exception('An error occurred'); + }); + + $failedPlatform = $this->createMock(PlatformInterface::class); + $failedPlatform->expects($this->any())->method('invoke') + ->willReturnCallback(static function (): DeferredResult { + static $call = 0; + + if (1 === ++$call) { + throw new \Exception('An error occurred from a failing platform'); + } + + throw new \Exception('An error occurred from a failing platform'); + }); + + $logger = $this->createMock(LoggerInterface::class); + $logger->expects($this->exactly(2))->method('error'); + + $failoverPlatform = new FailoverPlatform([ + $failedPlatform, + $firstPlatform, + ], clock: $clock, retryPeriod: 6, logger: $logger); + + $firstResult = $failoverPlatform->invoke('foo', 'foo'); + + $this->assertSame('foo', $firstResult->asText()); + + $clock->sleep(2); + + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage('All platforms failed.'); + $this->expectExceptionCode(0); + $failoverPlatform->invoke('foo', 'foo'); + } + + public function testPlatformCannotRetrieveModelCatalogWhileAllPlatformFailedDuringRetryPeriod() + { + $clock = new MonotonicClock(); + + $firstPlatform = $this->createMock(PlatformInterface::class); + $firstPlatform->expects($this->any())->method('getModelCatalog') + ->willReturnCallback(function (): ModelCatalogInterface { + static $call = 0; + + if (1 === ++$call) { + return new FallbackModelCatalog(); + } + + throw new \Exception('An error occurred'); + }); + + $failedPlatform = $this->createMock(PlatformInterface::class); + $failedPlatform->expects($this->any())->method('getModelCatalog') + ->willReturnCallback(static function (): ModelCatalogInterface { + static $call = 0; + + if (1 === ++$call) { + throw new \Exception('An error occurred from a failing platform'); + } + + throw new \Exception('An error occurred from a failing platform'); + }); + + $logger = $this->createMock(LoggerInterface::class); + $logger->expects($this->exactly(2))->method('error'); + + $failoverPlatform = new FailoverPlatform([ + $failedPlatform, + $firstPlatform, + ], clock: $clock, retryPeriod: 6, logger: $logger); + + $this->assertInstanceOf(FallbackModelCatalog::class, $failoverPlatform->getModelCatalog()); + + $clock->sleep(2); + + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage('All platforms failed.'); + $this->expectExceptionCode(0); + $failoverPlatform->getModelCatalog(); + } +}