diff --git a/README.md b/README.md index 0dbb72b..f37fe30 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,8 @@ The default benchmark uses the ImageNet validation set and reports RGB `uint8` decode throughput across common Python libraries and CPU families. +Preprint: [Single-Thread JPEG Decoder Benchmarks Mis-Evaluate ML Data Loaders](https://arxiv.org/abs/2605.08731). + ## Results The plots and tables below are generated from `output//*.json`. To refresh after a new run: @@ -277,16 +279,16 @@ See [CONTRIBUTING.md](CONTRIBUTING.md) for how to add a new decoder. ## Citation -If you found this work useful, please cite: +If you found this benchmark useful in your research or engineering work, please cite the preprint: ```bibtex -@misc{iglovikov2025speed, - title={Need for Speed: A Comprehensive Benchmark of JPEG Decoders in Python}, +@misc{iglovikov2026singlethreadjpegdecoderbenchmarks, + title={Single-Thread JPEG Decoder Benchmarks Mis-Evaluate ML Data Loaders}, author={Vladimir Iglovikov}, - year={2025}, - eprint={2501.13131}, + year={2026}, + eprint={2605.08731}, archivePrefix={arXiv}, - primaryClass={eess.IV}, - doi={10.48550/arXiv.2501.13131} + primaryClass={cs.PF}, + url={https://arxiv.org/abs/2605.08731}, } ``` diff --git a/tests/test_paper_assets.py b/tests/test_paper_assets.py index c31e0f1..f02d1e2 100644 --- a/tests/test_paper_assets.py +++ b/tests/test_paper_assets.py @@ -13,6 +13,7 @@ PAPER_PLATFORMS, ROBUSTNESS_DECODERS, WORKERS_ORDER, + generate_platform_recommendation_table, generate_robustness_table, validate_paper_data, ) @@ -135,3 +136,39 @@ def test_robustness_table_surfaces_only_observed_skip_decoders(tmp_path: Path) - expected_prefix = "1 / 50,000" if decoder in EXPECTED_SKIP_DECODERS else "0 / 50,000" assert expected_prefix in matching_line + + +def test_platform_recommendation_table_has_three_zero_skip_choices_per_platform(tmp_path: Path) -> None: + root = _repo_output() + dest = tmp_path / "table07_platform_recommendations.tex" + generate_platform_recommendation_table(load_dataloader_results(root), dest) + + text = dest.read_text(encoding="utf-8") + data_rows = [line for line in text.splitlines() if "GCP \\texttt" in line] + assert len(data_rows) == len(PAPER_PLATFORMS) + assert all(row.count("img/s") == 3 for row in data_rows) + + for decoder in EXPECTED_SKIP_DECODERS: + assert f"\\texttt{{{decoder}}}:" not in text + assert r"\texttt{pyvips}:" not in text + assert r"\texttt{tensorflow}:" not in text + + expected_choices = [ + r"\texttt{simplejpeg}: 1754 img/s ($w=8$)", + r"\texttt{opencv}: 1707 img/s ($w=8$)", + r"\texttt{imagecodecs}: 1677 img/s ($w=8$)", + r"\texttt{torchvision}: 1596 img/s ($w=8$)", + r"\texttt{imagecodecs}: 1543 img/s ($w=4$)", + r"\texttt{simplejpeg}: 1521 img/s ($w=4$)", + r"\texttt{torchvision}: 2920 img/s ($w=8$)", + r"\texttt{opencv}: 2814 img/s ($w=8$)", + r"\texttt{simplejpeg}: 2739 img/s ($w=8$)", + r"\texttt{imageio}: 2561 img/s ($w=8$)", + r"\texttt{torchvision}: 2557 img/s ($w=8$)", + r"\texttt{simplejpeg}: 2421 img/s ($w=8$)", + r"\texttt{simplejpeg}: 1557 img/s ($w=8$)", + r"\texttt{torchvision}: 1504 img/s ($w=8$)", + r"\texttt{imageio}: 1466 img/s ($w=8$)", + ] + for choice in expected_choices: + assert choice in text diff --git a/tools/paper_assets.py b/tools/paper_assets.py index 183a7dd..26c68e0 100644 --- a/tools/paper_assets.py +++ b/tools/paper_assets.py @@ -35,12 +35,15 @@ def _require_plotting() -> tuple[ModuleType, ModuleType, type]: if plt is not None and sns is not None and Patch is not None: return plt, sns, Patch try: + import matplotlib as mpl import matplotlib.pyplot as plt import seaborn as sns from matplotlib.patches import Patch except ImportError as exc: msg = "Figure generation requires matplotlib and seaborn. Install with `uv sync --extra plot`." raise RuntimeError(msg) from exc + mpl.rcParams["pdf.fonttype"] = 42 + mpl.rcParams["ps.fonttype"] = 42 return plt, sns, Patch @@ -154,6 +157,22 @@ def _fmt_ips(v: float | None) -> str: return f"{v:.0f}" +def _latex_escape(s: str) -> str: + replacements = { + "\\": r"\textbackslash{}", + "&": r"\&", + "%": r"\%", + "$": r"\$", + "#": r"\#", + "_": r"\_", + "{": r"\{", + "}": r"\}", + "~": r"\textasciitilde{}", + "^": r"\textasciicircum{}", + } + return "".join(replacements.get(ch, ch) for ch in s) + + def _validate_platform_set(label: str, actual: set[str]) -> None: expected_platforms = set(PAPER_PLATFORMS) if actual != expected_platforms: @@ -404,6 +423,83 @@ def generate_recommendation_table(dl: pd.DataFrame, dest: Path) -> None: ) +def _format_platform_choice(row: pd.Series) -> str: + lib = _latex_escape(str(row["library"])) + ips = float(row["peak_ips"]) + workers = int(row["peak_workers"]) + return rf"\texttt{{{lib}}}: {ips:.0f} img/s ($w={workers}$)" + + +def _platform_recommendation_rows(dl: pd.DataFrame) -> list[list[str]]: + dl = _paper_scope(dl) + peak = _peak_dataloader_rows(dl) + plat_labels = _platform_labels(_cpu_by_platform(dl)) + zero_skip_loader_decoders = EXPECTED_DATALOADER_DECODERS - EXPECTED_SKIP_DECODERS + + rows: list[list[str]] = [] + for plat in PAPER_PLATFORMS: + sub = peak[(peak["platform"] == plat) & (peak["library"].isin(zero_skip_loader_decoders))] + sub = sub.sort_values(["peak_ips", "library"], ascending=[False, True]) + if len(sub) < 3: + raise ValueError(f"{plat} has fewer than three zero-skip DataLoader choices") + choices = [_format_platform_choice(sub.iloc[i]) for i in range(3)] + rows.append( + [ + _latex_escape(plat_labels[plat]), + *choices, + rf"GCP \texttt{{{_latex_escape(PLATFORM_MACHINE[plat])}}}", + ], + ) + return rows + + +def generate_platform_recommendation_table(dl: pd.DataFrame, dest: Path) -> None: + headers = [ + "Platform", + "First zero-skip choice", + "Second zero-skip choice", + "Third zero-skip choice", + "Note", + ] + rows = _platform_recommendation_rows(dl) + body = "\n".join(" " + " & ".join(row) + r" \\" for row in rows) + header = " & ".join(headers) + caption = ( + r" \caption{Per-platform zero-skip DataLoader starting points. Values are measured peak PyTorch " + r"\texttt{DataLoader} throughput on the paper matrix; use them as initial guidance, not as a " + r"universal recommendation.}" + ) + note = ( + r" {\footnotesize Strict libjpeg-turbo-family wrappers may be fast but skipped one ImageNet JPEG; " + r"PyVips and TensorFlow are not PyTorch \texttt{DataLoader} choices in this harness.\par}" + ) + dest.write_text( + "\n".join( + [ + r"\begin{table}[ht]", + r" \centering", + r" \scriptsize", + r" \setlength{\tabcolsep}{2pt}", + r" \renewcommand{\arraystretch}{1.08}", + caption, + r" \label{tab:platform-starting-points}", + r" \begin{tabularx}{\linewidth}{@{}YYYYY@{}}", + r" \toprule", + f" {header} " + r"\\", + r" \midrule", + body, + r" \bottomrule", + r" \end{tabularx}", + r" \vspace{2pt}", + note, + r"\end{table}", + "", + ], + ), + encoding="utf-8", + ) + + def _raw_library_name(library: str) -> str: return "kornia" if library == "kornia-rs" else library @@ -779,6 +875,7 @@ def generate_tables(input_dir: Path, paper_dir: Path) -> None: generate_amd_w4_w8_table(dl, gen / "table04_amd_w4_w8.md") generate_robustness_table(input_dir, df_1t, gen / "table05_robustness.md") generate_recommendation_table(dl, gen / "table06_recommendation_tier.md") + generate_platform_recommendation_table(dl, gen / "table07_platform_recommendations.tex") (gen / "README.md").write_text( "# Generated paper tables\n\n" "- `table01_hardware.md`\n" @@ -786,7 +883,8 @@ def generate_tables(input_dir: Path, paper_dir: Path) -> None: "- `table03_peak_dataloader.md`\n" "- `table04_amd_w4_w8.md`\n" "- `table05_robustness.md`\n" - "- `table06_recommendation_tier.md`\n\n" + "- `table06_recommendation_tier.md`\n" + "- `table07_platform_recommendations.tex`\n\n" "Regenerate from repo root:\n\n" "```bash\n" "uv run --extra plot python -m tools.paper_assets --tables\n"