From 920e2648ba79018bc65731de6587a7887f26e17b Mon Sep 17 00:00:00 2001 From: "evan.mcelheny" Date: Fri, 14 Aug 2026 11:19:56 -0400 Subject: [PATCH 01/10] Add Celery task duration metrics Workspace: local --- lemur/common/celery.py | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index 195f8355f..1cea48e0a 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -18,6 +18,7 @@ after_setup_logger, after_setup_task_logger, task_failure, + task_prerun, task_received, task_revoked, task_success, @@ -53,6 +54,7 @@ flask_app = create_app() red = RedisHandler().redis() +_task_started_at = {} def make_celery(app): @@ -197,6 +199,17 @@ def report_celery_last_success_metrics(): metrics.send(f"{function}.success", "counter", 1) +@task_prerun.connect +def report_task_started(**kwargs): + """ + Record task start time so we can emit duration on completion/failure. + """ + with flask_app.app_context(): + task_id = kwargs.get("task_id") + if task_id: + _task_started_at[task_id] = time.monotonic() + + @task_received.connect def report_number_pending_tasks(**kwargs): """ @@ -213,6 +226,21 @@ def report_number_pending_tasks(**kwargs): ) +def _emit_task_duration(metric_status, **kwargs): + tags = get_celery_request_tags(**kwargs) + started_at = _task_started_at.pop(tags["task_id"], None) + if started_at is None: + return tags + duration_ms = int((time.monotonic() - started_at) * 1000) + metrics.send( + "celery.task_duration", + "TIMER", + duration_ms, + metric_tags={"task_name": tags["task_name"], "status": metric_status}, + ) + return tags + + @task_success.connect def report_successful_task(**kwargs): """ @@ -221,7 +249,7 @@ def report_successful_task(**kwargs): https://docs.celeryproject.org/en/latest/userguide/signals.html#task-success """ with flask_app.app_context(): - tags = get_celery_request_tags(**kwargs) + tags = _emit_task_duration("success", **kwargs) red.set(f"{tags['task_name']}.last_success", int(time.time())) metrics.send("celery.successful_task", "TIMER", 1, metric_tags=tags) # Emit failed_task=0 on success so the counter stays dense (0 when healthy) @@ -246,6 +274,7 @@ def report_failed_task(**kwargs): "function": f"{__name__}.{sys._getframe().f_code.co_name}", "Message": "Celery Task Failure", } + _emit_task_duration("failure", **kwargs) # Add traceback if exception info is in the kwargs einfo = kwargs.get("einfo") @@ -272,7 +301,7 @@ def report_revoked_task(**kwargs): "Message": "Celery Task Revoked", } - error_tags = get_celery_request_tags(**kwargs) + error_tags = _emit_task_duration("revoked", **kwargs) log_data.update(error_tags) current_app.logger.error(log_data) From 9294c179617b85e2ff60104c370ff5f5e64448ca Mon Sep 17 00:00:00 2001 From: "evan.mcelheny" Date: Fri, 14 Aug 2026 13:46:05 -0400 Subject: [PATCH 02/10] celery: clarify _emit_task_duration param name + docstring Workspace: local --- lemur/common/celery.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index 1cea48e0a..c1a124340 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -226,7 +226,13 @@ def report_number_pending_tasks(**kwargs): ) -def _emit_task_duration(metric_status, **kwargs): +def _emit_task_duration(status, **kwargs): + """ + Emit a task duration metric if we previously recorded a start time for the task. + + Returns the tags for the task (from get_celery_request_tags) so callers can continue + to use them for further metrics/logging. + """ tags = get_celery_request_tags(**kwargs) started_at = _task_started_at.pop(tags["task_id"], None) if started_at is None: @@ -236,11 +242,19 @@ def _emit_task_duration(metric_status, **kwargs): "celery.task_duration", "TIMER", duration_ms, - metric_tags={"task_name": tags["task_name"], "status": metric_status}, + metric_tags={"task_name": tags["task_name"], "status": status}, ) return tags +def _task_status_from_failure(**kwargs): + einfo = kwargs.get("einfo") + if einfo and getattr(getattr(einfo, "exception", None), "__class__", None): + if einfo.exception.__class__.__name__ == "SoftTimeLimitExceeded": + return "timeout" + return "failure" + + @task_success.connect def report_successful_task(**kwargs): """ @@ -274,7 +288,7 @@ def report_failed_task(**kwargs): "function": f"{__name__}.{sys._getframe().f_code.co_name}", "Message": "Celery Task Failure", } - _emit_task_duration("failure", **kwargs) + _emit_task_duration(_task_status_from_failure(**kwargs), **kwargs) # Add traceback if exception info is in the kwargs einfo = kwargs.get("einfo") From 7555b462ebdbc5a1adc7e4dbc9bb7241996eeafa Mon Sep 17 00:00:00 2001 From: "evan.mcelheny" Date: Fri, 14 Aug 2026 14:03:48 -0400 Subject: [PATCH 03/10] Add Celery task duration tests Workspace: local --- lemur/common/celery.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index c1a124340..cdf0c3b45 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -204,10 +204,9 @@ def report_task_started(**kwargs): """ Record task start time so we can emit duration on completion/failure. """ - with flask_app.app_context(): - task_id = kwargs.get("task_id") - if task_id: - _task_started_at[task_id] = time.monotonic() + task_id = kwargs.get("task_id") + if task_id: + _task_started_at[task_id] = time.monotonic() @task_received.connect @@ -249,9 +248,9 @@ def _emit_task_duration(status, **kwargs): def _task_status_from_failure(**kwargs): einfo = kwargs.get("einfo") - if einfo and getattr(getattr(einfo, "exception", None), "__class__", None): - if einfo.exception.__class__.__name__ == "SoftTimeLimitExceeded": - return "timeout" + exception = getattr(einfo, "exception", None) + if isinstance(exception, SoftTimeLimitExceeded): + return "timeout" return "failure" @@ -288,15 +287,13 @@ def report_failed_task(**kwargs): "function": f"{__name__}.{sys._getframe().f_code.co_name}", "Message": "Celery Task Failure", } - _emit_task_duration(_task_status_from_failure(**kwargs), **kwargs) + error_tags = _emit_task_duration(_task_status_from_failure(**kwargs), **kwargs) # Add traceback if exception info is in the kwargs einfo = kwargs.get("einfo") if einfo: log_data["traceback"] = einfo.traceback - error_tags = get_celery_request_tags(**kwargs) - log_data.update(error_tags) current_app.logger.error(log_data) metrics.send("celery.failed_task", "counter", 1, metric_tags=error_tags) From 9d7dd11b1219144a3b73e87dad3bdc3b142eaa67 Mon Sep 17 00:00:00 2001 From: "evan.mcelheny" Date: Fri, 14 Aug 2026 14:26:30 -0400 Subject: [PATCH 04/10] Add Celery task duration tests Workspace: local --- lemur/tests/test_celery_metrics.py | 81 ++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 lemur/tests/test_celery_metrics.py diff --git a/lemur/tests/test_celery_metrics.py b/lemur/tests/test_celery_metrics.py new file mode 100644 index 000000000..01470ef5b --- /dev/null +++ b/lemur/tests/test_celery_metrics.py @@ -0,0 +1,81 @@ +"""Tests for centralized Celery task duration metrics.""" +import sys +from unittest.mock import MagicMock, patch + +if "lemur.common.celery" not in sys.modules: + with patch("redis.StrictRedis") as _mock_redis: + _mock_redis.return_value.set.return_value = True + import lemur.common.celery # noqa: F401 + +import lemur.common.celery as _celery_module # noqa: E402 +_celery_module.current_app = MagicMock() + + +class FakeEinfo: + def __init__(self, exc): + self.exception = exc + self.traceback = "fake traceback" + + +class FakeRequest: + def __init__(self, task_id="task-1", name="lemur.common.celery.fake_task"): + self.id = task_id + self.hostname = "worker-1" + self.name = name + + +class FakeTask: + def __init__(self, task_id="task-1", name="lemur.common.celery.fake_task"): + self.request = FakeRequest(task_id=task_id, name=name) + self.hostname = "worker-1" + self.name = name + + +@patch("lemur.common.celery.metrics") +def test_task_duration_emitted_on_success_and_clears_start_time(mock_metrics): + _celery_module._task_started_at.clear() + _celery_module._task_started_at["task-1"] = 100.0 + + with patch("lemur.common.celery.time.monotonic", return_value=101.234), patch( + "lemur.common.celery.time.time", return_value=2000 + ): + fake_task = FakeTask() + _celery_module.report_successful_task(sender=fake_task, request=fake_task.request) + + duration_calls = [c for c in mock_metrics.send.call_args_list if c.args[0] == "celery.task_duration"] + assert len(duration_calls) == 1 + assert duration_calls[0].kwargs["metric_tags"] == {"task_name": "lemur.common.celery.fake_task", "status": "success"} + assert duration_calls[0].args[2] == 1233 + assert "task-1" not in _celery_module._task_started_at + + +@patch("lemur.common.celery.metrics") +def test_task_duration_emitted_on_failure_with_timeout_status(mock_metrics): + _celery_module._task_started_at.clear() + _celery_module._task_started_at["task-2"] = 50.0 + + with patch("lemur.common.celery.time.monotonic", return_value=51.5), patch( + "lemur.common.celery.time.time", return_value=2000 + ): + fake_task = FakeTask(task_id="task-2") + _celery_module.report_failed_task( + sender=fake_task, + request=fake_task.request, + einfo=FakeEinfo(_celery_module.SoftTimeLimitExceeded()), + ) + + duration_calls = [c for c in mock_metrics.send.call_args_list if c.args[0] == "celery.task_duration"] + assert len(duration_calls) == 1 + assert duration_calls[0].kwargs["metric_tags"]["status"] == "timeout" + assert duration_calls[0].args[2] == 1500 + assert "task-2" not in _celery_module._task_started_at + + +@patch("lemur.common.celery.metrics") +def test_task_duration_not_emitted_when_no_start_time(mock_metrics): + _celery_module._task_started_at.clear() + + fake_task = FakeTask(task_id="task-3") + _celery_module.report_successful_task(sender=fake_task, request=fake_task.request) + + assert not [c for c in mock_metrics.send.call_args_list if c.args[0] == "celery.task_duration"] From cf3fc100ffe10c5e245abc0c161e0615b9e7d088 Mon Sep 17 00:00:00 2001 From: "evan.mcelheny" Date: Fri, 14 Aug 2026 14:39:29 -0400 Subject: [PATCH 05/10] Fix Celery task duration tests Workspace: local --- lemur/tests/test_celery_metrics.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/lemur/tests/test_celery_metrics.py b/lemur/tests/test_celery_metrics.py index 01470ef5b..0f2cb7320 100644 --- a/lemur/tests/test_celery_metrics.py +++ b/lemur/tests/test_celery_metrics.py @@ -8,7 +8,6 @@ import lemur.common.celery # noqa: F401 import lemur.common.celery as _celery_module # noqa: E402 -_celery_module.current_app = MagicMock() class FakeEinfo: @@ -36,9 +35,9 @@ def test_task_duration_emitted_on_success_and_clears_start_time(mock_metrics): _celery_module._task_started_at.clear() _celery_module._task_started_at["task-1"] = 100.0 - with patch("lemur.common.celery.time.monotonic", return_value=101.234), patch( - "lemur.common.celery.time.time", return_value=2000 - ): + with patch.object(_celery_module, "current_app", MagicMock()), patch( + "lemur.common.celery.time.monotonic", return_value=101.234 + ), patch("lemur.common.celery.time.time", return_value=2000): fake_task = FakeTask() _celery_module.report_successful_task(sender=fake_task, request=fake_task.request) @@ -54,9 +53,9 @@ def test_task_duration_emitted_on_failure_with_timeout_status(mock_metrics): _celery_module._task_started_at.clear() _celery_module._task_started_at["task-2"] = 50.0 - with patch("lemur.common.celery.time.monotonic", return_value=51.5), patch( - "lemur.common.celery.time.time", return_value=2000 - ): + with patch.object(_celery_module, "current_app", MagicMock()), patch( + "lemur.common.celery.time.monotonic", return_value=51.5 + ), patch("lemur.common.celery.time.time", return_value=2000): fake_task = FakeTask(task_id="task-2") _celery_module.report_failed_task( sender=fake_task, @@ -75,7 +74,8 @@ def test_task_duration_emitted_on_failure_with_timeout_status(mock_metrics): def test_task_duration_not_emitted_when_no_start_time(mock_metrics): _celery_module._task_started_at.clear() - fake_task = FakeTask(task_id="task-3") - _celery_module.report_successful_task(sender=fake_task, request=fake_task.request) + with patch.object(_celery_module, "current_app", MagicMock()): + fake_task = FakeTask(task_id="task-3") + _celery_module.report_successful_task(sender=fake_task, request=fake_task.request) assert not [c for c in mock_metrics.send.call_args_list if c.args[0] == "celery.task_duration"] From 38a5f884e535dc52ef746267aea40a59bede093f Mon Sep 17 00:00:00 2001 From: "evan.mcelheny" Date: Mon, 17 Aug 2026 13:01:00 -0400 Subject: [PATCH 06/10] fix(logs): guard audit_log against detached g.current_user g.current_user can be a detached/expired SQLAlchemy User in some contexts (e.g. tests), so accessing .email raises DetachedInstanceError. Fall back to the anonymous LEMUR label instead of crashing. Fixes flaky CI failures in test_certificates.py and test_pending_certificates.py. Workspace: local --- lemur/logs/service.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/lemur/logs/service.py b/lemur/logs/service.py index 1890dcdbb..5baa9d218 100644 --- a/lemur/logs/service.py +++ b/lemur/logs/service.py @@ -9,6 +9,7 @@ """ from flask import current_app, g +from sqlalchemy.orm.exc import DetachedInstanceError from lemur import database from lemur.logs.models import Log @@ -48,7 +49,16 @@ def audit_log(action, entity, message): :return: """ - user = g.current_user.email if hasattr(g, "current_user") else "LEMUR" + current_user = getattr(g, "current_user", None) + if current_user is not None: + try: + user = current_user.email + except DetachedInstanceError: + # g.current_user may be a detached/expired instance in some + # contexts (e.g. tests); fall back to the anonymous label. + user = "LEMUR" + else: + user = "LEMUR" log_data = { "function": "lemur-audit", "action": action, From d0e7467b0a3a72c7e55c1be17704034aef1c9333 Mon Sep 17 00:00:00 2001 From: "evan.mcelheny" Date: Mon, 17 Aug 2026 13:10:56 -0400 Subject: [PATCH 07/10] Revert "fix(logs): guard audit_log against detached g.current_user" This reverts commit 38a5f884e535dc52ef746267aea40a59bede093f. Workspace: local --- lemur/logs/service.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/lemur/logs/service.py b/lemur/logs/service.py index 5baa9d218..1890dcdbb 100644 --- a/lemur/logs/service.py +++ b/lemur/logs/service.py @@ -9,7 +9,6 @@ """ from flask import current_app, g -from sqlalchemy.orm.exc import DetachedInstanceError from lemur import database from lemur.logs.models import Log @@ -49,16 +48,7 @@ def audit_log(action, entity, message): :return: """ - current_user = getattr(g, "current_user", None) - if current_user is not None: - try: - user = current_user.email - except DetachedInstanceError: - # g.current_user may be a detached/expired instance in some - # contexts (e.g. tests); fall back to the anonymous label. - user = "LEMUR" - else: - user = "LEMUR" + user = g.current_user.email if hasattr(g, "current_user") else "LEMUR" log_data = { "function": "lemur-audit", "action": action, From 0da97505c7aaae5b0fb882b9719a62b18a33df07 Mon Sep 17 00:00:00 2001 From: "evan.mcelheny" Date: Mon, 17 Aug 2026 13:10:59 -0400 Subject: [PATCH 08/10] test: clear g.current_user between tests to fix DetachedInstanceError flakiness g is app-context scoped and persists across tests, so a User set by an authenticated request in one test leaks into later tests as a stale, detached (and expired) instance. Accessing its attributes then raises DetachedInstanceError in audit_log, certificate schema serialization, etc. Clear g.current_user in the session fixture teardown so each test starts clean. Workspace: local --- lemur/tests/conftest.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lemur/tests/conftest.py b/lemur/tests/conftest.py index 899b91fb2..419ed91ee 100644 --- a/lemur/tests/conftest.py +++ b/lemur/tests/conftest.py @@ -5,7 +5,7 @@ from cryptography import x509 from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes -from flask import current_app +from flask import current_app, g from flask_principal import identity_changed, Identity from sqlalchemy.sql import text @@ -104,6 +104,11 @@ def session(db, request): db.session.begin_nested() yield db.session db.session.rollback() + # g is app-context scoped and persists across tests. Clear request-scoped + # state so a stale, detached User from an earlier test isn't reused by a + # later test (which raises DetachedInstanceError when its attributes are + # expired). + g.pop("current_user", None) @pytest.fixture(scope="function") From 4ce0348adf9651d1945f4a1d0393902e52c71399 Mon Sep 17 00:00:00 2001 From: "evan.mcelheny" Date: Mon, 17 Aug 2026 17:43:22 -0400 Subject: [PATCH 09/10] feat(celery): emit per-task .failed counter for renewal-pipeline SLO Adds a per-task failure counter (lemur.common.celery.{task}.failed) in the task_failure signal handler, symmetric with the existing per-task .success counter emitted at the end of each task. Together they drive the certificate renewal-pipeline SLO (success / (success + failed) per task). Low-cardinality task_name tag only; existing celery.failed_task signal unchanged. Workspace: local --- lemur/common/celery.py | 9 +++++++ lemur/tests/test_celery_metrics.py | 38 ++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index cdf0c3b45..2a7dc8243 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -297,6 +297,15 @@ def report_failed_task(**kwargs): log_data.update(error_tags) current_app.logger.error(log_data) metrics.send("celery.failed_task", "counter", 1, metric_tags=error_tags) + # Per-task failure counter, symmetric with the per-task `.success` counter + # emitted at the end of each task. Drives the renewal-pipeline SLO + # (success / (success + failed) per task). Low-cardinality tags only. + metrics.send( + f"{error_tags['task_name']}.failed", + "counter", + 1, + metric_tags={"task_name": error_tags["task_name"]}, + ) @task_revoked.connect diff --git a/lemur/tests/test_celery_metrics.py b/lemur/tests/test_celery_metrics.py index 0f2cb7320..4411612e8 100644 --- a/lemur/tests/test_celery_metrics.py +++ b/lemur/tests/test_celery_metrics.py @@ -79,3 +79,41 @@ def test_task_duration_not_emitted_when_no_start_time(mock_metrics): _celery_module.report_successful_task(sender=fake_task, request=fake_task.request) assert not [c for c in mock_metrics.send.call_args_list if c.args[0] == "celery.task_duration"] + + +@patch("lemur.common.celery.metrics") +def test_per_task_failed_counter_emitted_on_failure(mock_metrics): + _celery_module._task_started_at.clear() + + with patch.object(_celery_module, "current_app", MagicMock()): + fake_task = FakeTask(task_id="task-4", name="lemur.common.celery.certificate_reissue") + _celery_module.report_failed_task( + sender=fake_task, + request=fake_task.request, + einfo=FakeEinfo(RuntimeError("boom")), + ) + + failed_calls = [ + c for c in mock_metrics.send.call_args_list if c.args[0].endswith(".failed") + ] + assert len(failed_calls) == 1 + assert failed_calls[0].args[0] == "lemur.common.celery.certificate_reissue.failed" + assert failed_calls[0].kwargs["metric_tags"] == { + "task_name": "lemur.common.celery.certificate_reissue" + } + + +@patch("lemur.common.celery.metrics") +def test_per_task_failed_counter_absent_on_success(mock_metrics): + _celery_module._task_started_at.clear() + + with patch.object(_celery_module, "current_app", MagicMock()), patch( + "lemur.common.celery.time.time", return_value=2000 + ): + fake_task = FakeTask(task_id="task-5", name="lemur.common.celery.certificate_reissue") + _celery_module.report_successful_task(sender=fake_task, request=fake_task.request) + + failed_calls = [ + c for c in mock_metrics.send.call_args_list if c.args[0].endswith(".failed") + ] + assert failed_calls == [] From bb2e47abe8cb95781965cbe4a6ea64346516eee2 Mon Sep 17 00:00:00 2001 From: "evan.mcelheny" Date: Mon, 17 Aug 2026 17:50:56 -0400 Subject: [PATCH 10/10] Revert "feat(celery): emit per-task .failed counter for renewal-pipeline SLO" This reverts commit 4ce0348adf9651d1945f4a1d0393902e52c71399. Workspace: local --- lemur/common/celery.py | 9 ------- lemur/tests/test_celery_metrics.py | 38 ------------------------------ 2 files changed, 47 deletions(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index 2a7dc8243..cdf0c3b45 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -297,15 +297,6 @@ def report_failed_task(**kwargs): log_data.update(error_tags) current_app.logger.error(log_data) metrics.send("celery.failed_task", "counter", 1, metric_tags=error_tags) - # Per-task failure counter, symmetric with the per-task `.success` counter - # emitted at the end of each task. Drives the renewal-pipeline SLO - # (success / (success + failed) per task). Low-cardinality tags only. - metrics.send( - f"{error_tags['task_name']}.failed", - "counter", - 1, - metric_tags={"task_name": error_tags["task_name"]}, - ) @task_revoked.connect diff --git a/lemur/tests/test_celery_metrics.py b/lemur/tests/test_celery_metrics.py index 4411612e8..0f2cb7320 100644 --- a/lemur/tests/test_celery_metrics.py +++ b/lemur/tests/test_celery_metrics.py @@ -79,41 +79,3 @@ def test_task_duration_not_emitted_when_no_start_time(mock_metrics): _celery_module.report_successful_task(sender=fake_task, request=fake_task.request) assert not [c for c in mock_metrics.send.call_args_list if c.args[0] == "celery.task_duration"] - - -@patch("lemur.common.celery.metrics") -def test_per_task_failed_counter_emitted_on_failure(mock_metrics): - _celery_module._task_started_at.clear() - - with patch.object(_celery_module, "current_app", MagicMock()): - fake_task = FakeTask(task_id="task-4", name="lemur.common.celery.certificate_reissue") - _celery_module.report_failed_task( - sender=fake_task, - request=fake_task.request, - einfo=FakeEinfo(RuntimeError("boom")), - ) - - failed_calls = [ - c for c in mock_metrics.send.call_args_list if c.args[0].endswith(".failed") - ] - assert len(failed_calls) == 1 - assert failed_calls[0].args[0] == "lemur.common.celery.certificate_reissue.failed" - assert failed_calls[0].kwargs["metric_tags"] == { - "task_name": "lemur.common.celery.certificate_reissue" - } - - -@patch("lemur.common.celery.metrics") -def test_per_task_failed_counter_absent_on_success(mock_metrics): - _celery_module._task_started_at.clear() - - with patch.object(_celery_module, "current_app", MagicMock()), patch( - "lemur.common.celery.time.time", return_value=2000 - ): - fake_task = FakeTask(task_id="task-5", name="lemur.common.celery.certificate_reissue") - _celery_module.report_successful_task(sender=fake_task, request=fake_task.request) - - failed_calls = [ - c for c in mock_metrics.send.call_args_list if c.args[0].endswith(".failed") - ] - assert failed_calls == []