From 2a1262190118f4d689895eb0643aafb5f1470976 Mon Sep 17 00:00:00 2001 From: Ahmad El Youssef Date: Mon, 29 Jun 2026 11:17:50 -0700 Subject: [PATCH] SparkBench 3x: change default number of shuffle partitions to 3x NPROC (#692) Summary: SparkBench 3x: change default number of shuffle partitions to 3x NPROC Reviewed By: excelle08 Differential Revision: D109623179 --- benchpress/config/jobs.yml | 2 +- .../templates/proj_root/scripts/run_perf_common.py | 8 ++++---- packages/spark_standalone/templates/runner.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/benchpress/config/jobs.yml b/benchpress/config/jobs.yml index 57aede48a..c0ee5c56a 100644 --- a/benchpress/config/jobs.yml +++ b/benchpress/config/jobs.yml @@ -1042,7 +1042,7 @@ - benchmark: spark_standalone name: spark_standalone_remote_3x - description: Spark standalone using remote SSDs with 3x query load (2 more joins) and auto-scaled shuffle partitions + description: Spark standalone using remote SSDs with 3x query load (2 more joins) and auto-scaled shuffle partitions (3 * CPU cores) args: - 'run' - '--dataset-path /flash23/' diff --git a/packages/spark_standalone/templates/proj_root/scripts/run_perf_common.py b/packages/spark_standalone/templates/proj_root/scripts/run_perf_common.py index bb0fd44f7..842be3b4b 100755 --- a/packages/spark_standalone/templates/proj_root/scripts/run_perf_common.py +++ b/packages/spark_standalone/templates/proj_root/scripts/run_perf_common.py @@ -93,11 +93,11 @@ def write_sql_create_tables(args) -> List[str]: # Set spark.sql.shuffle.partitions if specified if hasattr(args, "shuffle_partitions") and args.shuffle_partitions is not None: shuffle_partitions = args.shuffle_partitions - # Auto-scale to 4 * CPU cores if set to 0 or negative + # Auto-scale to 3 * CPU cores if set to 0 or negative if shuffle_partitions <= 0: - shuffle_partitions = 4 * os.cpu_count() + shuffle_partitions = 3 * os.cpu_count() print( - f"Auto-scaling shuffle partitions to {shuffle_partitions} (4 * {os.cpu_count()} CPU cores)" + f"Auto-scaling shuffle partitions to {shuffle_partitions} (3 * {os.cpu_count()} CPU cores)" ) fp.write(f"""SET spark.sql.shuffle.partitions = {shuffle_partitions};\n""") fp.write(f"""USE {args.database};""") @@ -631,7 +631,7 @@ def init_parser(): type=int, default=None, help="Set the number of partitions for Spark SQL shuffle operations. " - "If set to 0 or negative, automatically scales to 4 * number of available CPU cores.", + "If set to 0 or negative, automatically scales to 3 * number of available CPU cores.", ) # custom query option (for run_parser and exp_parser since they run queries) for x in [run_parser, exp_parser]: diff --git a/packages/spark_standalone/templates/runner.py b/packages/spark_standalone/templates/runner.py index 6469dafb1..2e8dfa403 100755 --- a/packages/spark_standalone/templates/runner.py +++ b/packages/spark_standalone/templates/runner.py @@ -427,7 +427,7 @@ def init_parser(): type=int, default=None, help="Set the number of partitions for Spark SQL shuffle operations. " - "If set to 0 or negative, automatically scales to 4 * number of available CPU cores.", + "If set to 0 or negative, automatically scales to 3 * number of available CPU cores.", ) run_parser.add_argument( "--query",