Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 9 additions & 7 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@

The default benchmark uses the ImageNet validation set and reports RGB `uint8` decode throughput across common Python libraries and CPU families.

Preprint: [Single-Thread JPEG Decoder Benchmarks Mis-Evaluate ML Data Loaders](https://arxiv.org/abs/2605.08731).

## Results

The plots and tables below are generated from `output/<platform>/*.json`. To refresh after a new run:
Expand Down Expand Up @@ -277,16 +279,16 @@ See [CONTRIBUTING.md](CONTRIBUTING.md) for how to add a new decoder.

## Citation

If you found this work useful, please cite:
If you found this benchmark useful in your research or engineering work, please cite the preprint:

```bibtex
@misc{iglovikov2025speed,
title={Need for Speed: A Comprehensive Benchmark of JPEG Decoders in Python},
@misc{iglovikov2026singlethreadjpegdecoderbenchmarks,
title={Single-Thread JPEG Decoder Benchmarks Mis-Evaluate ML Data Loaders},
author={Vladimir Iglovikov},
year={2025},
eprint={2501.13131},
year={2026},
eprint={2605.08731},
archivePrefix={arXiv},
primaryClass={eess.IV},
doi={10.48550/arXiv.2501.13131}
primaryClass={cs.PF},
url={https://arxiv.org/abs/2605.08731},
}
```
37 changes: 37 additions & 0 deletions tests/test_paper_assets.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
PAPER_PLATFORMS,
ROBUSTNESS_DECODERS,
WORKERS_ORDER,
generate_platform_recommendation_table,
generate_robustness_table,
validate_paper_data,
)
Expand Down Expand Up @@ -135,3 +136,39 @@ def test_robustness_table_surfaces_only_observed_skip_decoders(tmp_path: Path) -

expected_prefix = "1 / 50,000" if decoder in EXPECTED_SKIP_DECODERS else "0 / 50,000"
assert expected_prefix in matching_line


def test_platform_recommendation_table_has_three_zero_skip_choices_per_platform(tmp_path: Path) -> None:
root = _repo_output()
dest = tmp_path / "table07_platform_recommendations.tex"
generate_platform_recommendation_table(load_dataloader_results(root), dest)

text = dest.read_text(encoding="utf-8")
data_rows = [line for line in text.splitlines() if "GCP \\texttt" in line]
assert len(data_rows) == len(PAPER_PLATFORMS)
assert all(row.count("img/s") == 3 for row in data_rows)

Comment on lines +141 to +150

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

issue (testing): Missing negative-path test for the case where a platform has fewer than three zero-skip DataLoader choices.

The new test only exercises the happy path with real benchmark data and doesn’t verify the error branch in _platform_recommendation_rows that raises ValueError when a platform has fewer than three zero-skip choices.

Please add a small synthetic test that builds a minimal dl where one platform has only 0–2 zero-skip choices (or temporarily patches EXPECTED_DATALOADER_DECODERS / PAPER_PLATFORMS), and asserts that _platform_recommendation_rows or generate_platform_recommendation_table raises ValueError with the expected message. This will lock in the failure mode and protect future refactors.

for decoder in EXPECTED_SKIP_DECODERS:
assert f"\\texttt{{{decoder}}}:" not in text
assert r"\texttt{pyvips}:" not in text
assert r"\texttt{tensorflow}:" not in text

expected_choices = [
r"\texttt{simplejpeg}: 1754 img/s ($w=8$)",
r"\texttt{opencv}: 1707 img/s ($w=8$)",
r"\texttt{imagecodecs}: 1677 img/s ($w=8$)",
r"\texttt{torchvision}: 1596 img/s ($w=8$)",
r"\texttt{imagecodecs}: 1543 img/s ($w=4$)",
r"\texttt{simplejpeg}: 1521 img/s ($w=4$)",
r"\texttt{torchvision}: 2920 img/s ($w=8$)",
r"\texttt{opencv}: 2814 img/s ($w=8$)",
r"\texttt{simplejpeg}: 2739 img/s ($w=8$)",
r"\texttt{imageio}: 2561 img/s ($w=8$)",
r"\texttt{torchvision}: 2557 img/s ($w=8$)",
r"\texttt{simplejpeg}: 2421 img/s ($w=8$)",
r"\texttt{simplejpeg}: 1557 img/s ($w=8$)",
r"\texttt{torchvision}: 1504 img/s ($w=8$)",
r"\texttt{imageio}: 1466 img/s ($w=8$)",
]
for choice in expected_choices:
assert choice in text
100 changes: 99 additions & 1 deletion tools/paper_assets.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,12 +35,15 @@ def _require_plotting() -> tuple[ModuleType, ModuleType, type]:
if plt is not None and sns is not None and Patch is not None:
return plt, sns, Patch
try:
import matplotlib as mpl
import matplotlib.pyplot as plt
import seaborn as sns
from matplotlib.patches import Patch
except ImportError as exc:
msg = "Figure generation requires matplotlib and seaborn. Install with `uv sync --extra plot`."
raise RuntimeError(msg) from exc
mpl.rcParams["pdf.fonttype"] = 42
mpl.rcParams["ps.fonttype"] = 42
return plt, sns, Patch


Expand Down Expand Up @@ -154,6 +157,22 @@ def _fmt_ips(v: float | None) -> str:
return f"{v:.0f}"


def _latex_escape(s: str) -> str:
replacements = {
"\\": r"\textbackslash{}",
"&": r"\&",
"%": r"\%",
"$": r"\$",
"#": r"\#",
"_": r"\_",
"{": r"\{",
"}": r"\}",
"~": r"\textasciitilde{}",
"^": r"\textasciicircum{}",
}
return "".join(replacements.get(ch, ch) for ch in s)


def _validate_platform_set(label: str, actual: set[str]) -> None:
expected_platforms = set(PAPER_PLATFORMS)
if actual != expected_platforms:
Expand Down Expand Up @@ -404,6 +423,83 @@ def generate_recommendation_table(dl: pd.DataFrame, dest: Path) -> None:
)


def _format_platform_choice(row: pd.Series) -> str:
lib = _latex_escape(str(row["library"]))
ips = float(row["peak_ips"])
workers = int(row["peak_workers"])
return rf"\texttt{{{lib}}}: {ips:.0f} img/s ($w={workers}$)"


def _platform_recommendation_rows(dl: pd.DataFrame) -> list[list[str]]:
dl = _paper_scope(dl)
peak = _peak_dataloader_rows(dl)
plat_labels = _platform_labels(_cpu_by_platform(dl))
zero_skip_loader_decoders = EXPECTED_DATALOADER_DECODERS - EXPECTED_SKIP_DECODERS

rows: list[list[str]] = []
for plat in PAPER_PLATFORMS:
sub = peak[(peak["platform"] == plat) & (peak["library"].isin(zero_skip_loader_decoders))]
sub = sub.sort_values(["peak_ips", "library"], ascending=[False, True])
if len(sub) < 3:
raise ValueError(f"{plat} has fewer than three zero-skip DataLoader choices")
choices = [_format_platform_choice(sub.iloc[i]) for i in range(3)]
rows.append(
[
_latex_escape(plat_labels[plat]),
*choices,
rf"GCP \texttt{{{_latex_escape(PLATFORM_MACHINE[plat])}}}",
],
)
return rows


def generate_platform_recommendation_table(dl: pd.DataFrame, dest: Path) -> None:
headers = [
"Platform",
"First zero-skip choice",
"Second zero-skip choice",
"Third zero-skip choice",
"Note",
]
rows = _platform_recommendation_rows(dl)
body = "\n".join(" " + " & ".join(row) + r" \\" for row in rows)
header = " & ".join(headers)
caption = (
r" \caption{Per-platform zero-skip DataLoader starting points. Values are measured peak PyTorch "
r"\texttt{DataLoader} throughput on the paper matrix; use them as initial guidance, not as a "
r"universal recommendation.}"
)
note = (
r" {\footnotesize Strict libjpeg-turbo-family wrappers may be fast but skipped one ImageNet JPEG; "
r"PyVips and TensorFlow are not PyTorch \texttt{DataLoader} choices in this harness.\par}"
)
dest.write_text(
"\n".join(
[
r"\begin{table}[ht]",
r" \centering",
r" \scriptsize",
r" \setlength{\tabcolsep}{2pt}",
r" \renewcommand{\arraystretch}{1.08}",
caption,
r" \label{tab:platform-starting-points}",
r" \begin{tabularx}{\linewidth}{@{}YYYYY@{}}",
r" \toprule",
f" {header} " + r"\\",
r" \midrule",
body,
r" \bottomrule",
r" \end{tabularx}",
r" \vspace{2pt}",
note,
r"\end{table}",
"",
],
),
encoding="utf-8",
)


def _raw_library_name(library: str) -> str:
return "kornia" if library == "kornia-rs" else library

Expand Down Expand Up @@ -779,14 +875,16 @@ def generate_tables(input_dir: Path, paper_dir: Path) -> None:
generate_amd_w4_w8_table(dl, gen / "table04_amd_w4_w8.md")
generate_robustness_table(input_dir, df_1t, gen / "table05_robustness.md")
generate_recommendation_table(dl, gen / "table06_recommendation_tier.md")
generate_platform_recommendation_table(dl, gen / "table07_platform_recommendations.tex")
(gen / "README.md").write_text(
"# Generated paper tables\n\n"
"- `table01_hardware.md`\n"
"- `table02_single_thread.md`\n"
"- `table03_peak_dataloader.md`\n"
"- `table04_amd_w4_w8.md`\n"
"- `table05_robustness.md`\n"
"- `table06_recommendation_tier.md`\n\n"
"- `table06_recommendation_tier.md`\n"
"- `table07_platform_recommendations.tex`\n\n"
"Regenerate from repo root:\n\n"
"```bash\n"
"uv run --extra plot python -m tools.paper_assets --tables\n"
Expand Down
Loading