Skip to content

test: harden platform recommendation coverage #36

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ternaus merged 1 commit into from
May 12, 2026
Merged

test: harden platform recommendation coverage #36

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
108 changes: 89 additions & 19 deletions tests/test_paper_assets.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import pandas as pd
import pytest

from tools import paper_assets
from tools._results import load_dataloader_results, load_results
from tools.paper_assets import (
EXPECTED_DATALOADER_DECODERS,
Expand All @@ -13,6 +14,9 @@
PAPER_PLATFORMS,
ROBUSTNESS_DECODERS,
WORKERS_ORDER,
_latex_table_row,
_platform_recommendation_choices,
_platform_recommendation_rows,
generate_platform_recommendation_table,
generate_robustness_table,
validate_paper_data,
Expand Down Expand Up @@ -153,22 +157,88 @@ def test_platform_recommendation_table_has_three_zero_skip_choices_per_platform(
assert r"\texttt{pyvips}:" not in text
assert r"\texttt{tensorflow}:" not in text

expected_choices = [
r"\texttt{simplejpeg}: 1754 img/s ($w=8$)",
r"\texttt{opencv}: 1707 img/s ($w=8$)",
r"\texttt{imagecodecs}: 1677 img/s ($w=8$)",
r"\texttt{torchvision}: 1596 img/s ($w=8$)",
r"\texttt{imagecodecs}: 1543 img/s ($w=4$)",
r"\texttt{simplejpeg}: 1521 img/s ($w=4$)",
r"\texttt{torchvision}: 2920 img/s ($w=8$)",
r"\texttt{opencv}: 2814 img/s ($w=8$)",
r"\texttt{simplejpeg}: 2739 img/s ($w=8$)",
r"\texttt{imageio}: 2561 img/s ($w=8$)",
r"\texttt{torchvision}: 2557 img/s ($w=8$)",
r"\texttt{simplejpeg}: 2421 img/s ($w=8$)",
r"\texttt{simplejpeg}: 1557 img/s ($w=8$)",
r"\texttt{torchvision}: 1504 img/s ($w=8$)",
r"\texttt{imageio}: 1466 img/s ($w=8$)",
]
for choice in expected_choices:
assert choice in text
choices_by_platform = _platform_recommendation_choices(load_dataloader_results(root))
assert [platform for platform, _ in choices_by_platform] == list(PAPER_PLATFORMS)
for _, choices in choices_by_platform:
assert len(choices) == 3
assert choices["library"].is_unique
assert choices["peak_ips"].tolist() == sorted(choices["peak_ips"], reverse=True)
assert choices["peak_workers"].isin(WORKERS_ORDER).all()


def test_latex_table_row_formats_cells_with_indentation_and_linebreak() -> None:
assert _latex_table_row(["A", "B", "C"]) == r" A & B & C \\"


def test_platform_recommendation_rows_requires_three_zero_skip_choices() -> None:
platform = PAPER_PLATFORMS[0]
dl = pd.DataFrame(
[
{
"platform": platform,
"library": "opencv",
"num_workers": 0,
"images_per_second": 100.0,
"cpu_brand": "Synthetic CPU",
Comment on lines +173 to +182

@sourcery-ai sourcery-ai Bot May 12, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

suggestion (testing): Add a positive test that explicitly checks non-zero-skip rows are ignored when enough zero-skip rows exist.

To complement this negative-path test, please add a positive-path unit test that builds a small DataFrame with at least three zero-skip entries plus some higher-throughput non-zero-skip entries, and asserts that _platform_recommendation_rows selects only the zero-skip rows. This will lock in the intended filtering behavior and guard against regressions where non-zero-skip rows start being chosen.

},
{
"platform": platform,
"library": "imageio",
"num_workers": 2,
"images_per_second": 90.0,
"cpu_brand": "Synthetic CPU",
},
],
)

with pytest.raises(ValueError, match="fewer than three zero-skip DataLoader choices"):
_platform_recommendation_rows(dl)


def test_platform_recommendation_choices_ignore_skip_decoders(monkeypatch: pytest.MonkeyPatch) -> None:
platform = PAPER_PLATFORMS[0]
monkeypatch.setattr(paper_assets, "PAPER_PLATFORMS", (platform,))
dl = pd.DataFrame(
[
{
"platform": platform,
"library": "jpeg4py",
"num_workers": 8,
"images_per_second": 500.0,
"cpu_brand": "Synthetic CPU",
},
{
"platform": platform,
"library": "turbojpeg",
"num_workers": 8,
"images_per_second": 450.0,
"cpu_brand": "Synthetic CPU",
},
{
"platform": platform,
"library": "opencv",
"num_workers": 4,
"images_per_second": 300.0,
"cpu_brand": "Synthetic CPU",
},
{
"platform": platform,
"library": "imageio",
"num_workers": 2,
"images_per_second": 200.0,
"cpu_brand": "Synthetic CPU",
},
{
"platform": platform,
"library": "pillow",
"num_workers": 0,
"images_per_second": 100.0,
"cpu_brand": "Synthetic CPU",
},
],
)

[(selected_platform, choices)] = _platform_recommendation_choices(dl)

assert selected_platform == platform
assert choices["library"].tolist() == ["opencv", "imageio", "pillow"]
22 changes: 17 additions & 5 deletions tools/paper_assets.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -430,19 +430,27 @@ def _format_platform_choice(row: pd.Series) -> str:
return rf"\texttt{{{lib}}}: {ips:.0f} img/s ($w={workers}$)"


def _platform_recommendation_rows(dl: pd.DataFrame) -> list[list[str]]:
def _platform_recommendation_choices(dl: pd.DataFrame) -> list[tuple[str, pd.DataFrame]]:
dl = _paper_scope(dl)
peak = _peak_dataloader_rows(dl)
plat_labels = _platform_labels(_cpu_by_platform(dl))
zero_skip_loader_decoders = EXPECTED_DATALOADER_DECODERS - EXPECTED_SKIP_DECODERS

rows: list[list[str]] = []
choices_by_platform: list[tuple[str, pd.DataFrame]] = []
for plat in PAPER_PLATFORMS:
sub = peak[(peak["platform"] == plat) & (peak["library"].isin(zero_skip_loader_decoders))]
sub = sub.sort_values(["peak_ips", "library"], ascending=[False, True])
if len(sub) < 3:
raise ValueError(f"{plat} has fewer than three zero-skip DataLoader choices")
choices = [_format_platform_choice(sub.iloc[i]) for i in range(3)]
choices_by_platform.append((plat, sub.head(3).reset_index(drop=True)))
return choices_by_platform


def _platform_recommendation_rows(dl: pd.DataFrame) -> list[list[str]]:
plat_labels = _platform_labels(_cpu_by_platform(_paper_scope(dl)))

rows: list[list[str]] = []
for plat, choices_df in _platform_recommendation_choices(dl):
choices = [_format_platform_choice(choices_df.iloc[i]) for i in range(3)]
rows.append(
[
_latex_escape(plat_labels[plat]),
Expand All @@ -453,6 +461,10 @@ def _platform_recommendation_rows(dl: pd.DataFrame) -> list[list[str]]:
return rows


def _latex_table_row(row: list[str]) -> str:
return " " + " & ".join(row) + r" \\"


def generate_platform_recommendation_table(dl: pd.DataFrame, dest: Path) -> None:
headers = [
"Platform",
Expand All @@ -462,7 +474,7 @@ def generate_platform_recommendation_table(dl: pd.DataFrame, dest: Path) -> None
"Note",
]
rows = _platform_recommendation_rows(dl)
body = "\n".join(" " + " & ".join(row) + r" \\" for row in rows)
body = "\n".join(_latex_table_row(row) for row in rows)
header = " & ".join(headers)
caption = (
r" \caption{Per-platform zero-skip DataLoader starting points. Values are measured peak PyTorch "
Expand Down
Loading