Update scripts / readme

Author: alamb

datafusion-partitioned/README.md

@@ -27,6 +27,7 @@ The benchmark should be completed in under an hour. On-demand pricing is $0.6 pe
     ```
 
 1. `bash benchmark.sh`
+1. `./save-result.sh c6a.4xlarge`
 
 ### Know Issues
 

datafusion-partitioned/benchmark.sh

@@ -11,8 +11,8 @@ sudo apt-get update -y
 sudo apt-get install -y gcc
 
 echo "Install DataFusion main branch"
-git clone https://github.com/apache/arrow-datafusion.git
-cd arrow-datafusion/
+git clone https://github.com/apache/datafusion.git
+cd datafusion/
 git checkout 51.0.0
 CARGO_PROFILE_RELEASE_LTO=true RUSTFLAGS="-C codegen-units=1" cargo build --release --package datafusion-cli --bin datafusion-cli
 export PATH="`pwd`/target/release:$PATH"

datafusion-partitioned/save-result.sh

@@ -7,28 +7,7 @@
 #
 # example (save results/c6a.4xlarge.json)
 #         ./save-result.sh c6a.4xlarge
-#
-# Example input result.csv, showing three runs for query 1:
-# 1,1,0.108
-# 1,2,0.108
-# 1,3,0.108
-#
-# Example output output.json, stored in results/<machine.json
-# {
-#    "system":   "DataFusion (Parquet, partitioned)",
-#    "date": "2025-07-11",
-#    "machine": "c6a.xlarge",
-#    "cluster_size": 1,
-#    "proprietary":   "no",
-#    "tuned":   "no",
-#    "tags":   ["Rust","column-oriented","embedded","stateless", "lukewarm-cold-run"],
-#    "load_time": 0,
-#    "data_size": 14737666736,
-#    "result": [
-#        [0.108, 0.108, 0.108],
-#        ...
-#    ]
-# }
+
 MACHINE=$1
 OUTPUT_FILE="results/${MACHINE}.json"
 SYSTEM_NAME="DataFusion (Parquet, partitioned)"

datafusion/README.md

@@ -27,6 +27,7 @@ The benchmark should be completed in under an hour. On-demand pricing is $0.6 pe
     ```
 
 1. `bash benchmark.sh`
+1. `./save-result.sh c6a.4xlarge`
 
 ### Know Issues
 

datafusion/benchmark.sh

@@ -11,8 +11,8 @@ sudo apt-get update -y
 sudo apt-get install -y gcc
 
 echo "Install DataFusion main branch"
-git clone https://github.com/apache/arrow-datafusion.git
-cd arrow-datafusion/
+git clone https://github.com/apache/datafusion.git
+cd datafusion/
 git checkout 51.0.0
 CARGO_PROFILE_RELEASE_LTO=true RUSTFLAGS="-C codegen-units=1" cargo build --release --package datafusion-cli --bin datafusion-cli
 export PATH="`pwd`/target/release:$PATH"

datafusion/save-result.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# This scripts converts the raw results.csv data from `benchmark.sh` into a the
+# final json format used by the benchmark dashboard.
+#
+# usage : ./save-result.sh <machine>
+#
+# example (save results/c6a.4xlarge.json)
+#         ./save-result.sh c6a.4xlarge
+
+MACHINE=$1
+OUTPUT_FILE="results/${MACHINE}.json"
+SYSTEM_NAME="DataFusion (Parquet, single)"
+DATE=$(date +%Y-%m-%d)
+
+
+# Read the CSV and build the result array using sed
+RESULT_ARRAY=$(awk -F, '{arr[$1]=arr[$1]","$3} END {for (i=1;i<=length(arr);i++) {gsub(/^,/, "", arr[i]); printf "        ["arr[i]"]"; if (i<length(arr)) printf ",\n"}}' result.csv)
+
+# form the final JSON structure from the template
+cat <<EOF > $OUTPUT_FILE
+{
+    "system": "$SYSTEM_NAME",
+    "date": "$DATE",
+    "machine": "$MACHINE",
+    "cluster_size": 1,
+    "proprietary": "no",
+    "tuned": "no",
+    "tags": ["Rust","column-oriented","embedded","stateless"],
+    "load_time": 0,
+    "data_size": 14779976446,
+    "result": [
+        $RESULT_ARRAY
+    ]
+}
+EOF