GH-35460: Use simdjson instead of RapidJSON

ci/conan/all/conanfile.py

@@ -163,7 +163,7 @@ def configure(self):
     def layout(self):
         cmake_layout(self, src_folder="src")
 
-    def _requires_rapidjson(self):
+    def _requires_json_library(self):
         return self.options.with_json or self.options.encryption
 
     def requirements(self):
@@ -185,8 +185,9 @@ def requirements(self):
             self.requires("google-cloud-cpp/1.40.1")
         if self.options.with_grpc:
             self.requires("grpc/1.50.0")
-        if self._requires_rapidjson():
+        if self._requires_json_library():
             self.requires("rapidjson/1.1.0")
+            self.requires("simdjson/4.2.2")
         if self.options.with_llvm:
             self.requires("llvm-core/13.0.0")
         if self.options.with_openssl:
@@ -371,6 +372,7 @@ def generate(self):
             tc.variables["ARROW_LZ4_USE_SHARED"] = bool(self.dependencies["lz4"].options.shared)
         tc.variables["ARROW_WITH_SNAPPY"] = bool(self.options.with_snappy)
         tc.variables["RapidJSON_SOURCE"] = "SYSTEM"
+        tc.variables["SIMDJSON_SOURCE"] = "SYSTEM"
         tc.variables["Snappy_SOURCE"] = "SYSTEM"
         if self.options.with_snappy:
             tc.variables["ARROW_SNAPPY_USE_SHARED"] = bool(self.dependencies["snappy"].options.shared)
@@ -559,8 +561,9 @@ def package_info(self):
             self.cpp_info.components["libarrow"].requires.append("libbacktrace::libbacktrace")
         if self.options.with_cuda:
             self.cpp_info.components["libarrow"].requires.append("cuda::cuda")
-        if self._requires_rapidjson():
+        if self._requires_json_library():
             self.cpp_info.components["libarrow"].requires.append("rapidjson::rapidjson")
+            self.cpp_info.components["libarrow"].requires.append("simdjson::simdjson")
         if self.options.with_s3:
             # https://github.com/apache/arrow/blob/6b268f62a8a172249ef35f093009c740c32e1f36/cpp/src/arrow/CMakeLists.txt#L98
             self.cpp_info.components["libarrow"].requires.extend([f"aws-sdk-cpp::{x}" for x in ["cognito-identity", "core", "identity-management", "s3", "sts"]])

ci/docker/alpine-linux-3.22-cpp.dockerfile

@@ -59,6 +59,7 @@ RUN apk add \
         re2-dev \
         rsync \
         samurai \
+        simdjson-dev \
         snappy-dev \
         sqlite-dev \
         thrift-dev \

ci/docker/debian-12-cpp.dockerfile

@@ -71,6 +71,7 @@ RUN apt-get update -y -q && \
         libpsl-dev \
         libre2-dev \
         librtmp-dev \
+        libsimdjson-dev \
         libsnappy-dev \
         libsqlite3-dev \
         libssh-dev \

ci/docker/debian-13-cpp.dockerfile

@@ -71,6 +71,7 @@ RUN apt-get update -y -q && \
         libpsl-dev \
         libre2-dev \
         librtmp-dev \
+        libsimdjson-dev \
         libsnappy-dev \
         libsqlite3-dev \
         libssh-dev \

ci/docker/debian-experimental-cpp.dockerfile

@@ -65,6 +65,7 @@ RUN if [ -n "${gcc}" ]; then \
         libpsl-dev \
         libre2-dev \
         librtmp-dev \
+        libsimdjson-dev \
         libsnappy-dev \
         libsqlite3-dev \
         libssh-dev \

ci/docker/fedora-42-cpp.dockerfile

@@ -60,6 +60,7 @@ RUN dnf update -y && \
         python-pip \
         rapidjson-devel \
         re2-devel \
+        simdjson-dev \
         snappy-devel \
         thrift-devel \
         utf8proc-devel \

ci/docker/ubuntu-22.04-cpp.dockerfile

@@ -93,6 +93,7 @@ RUN apt-get update -y -q && \
         libradospp-dev \
         libre2-dev \
         librtmp-dev \
+        libsimdjson-dev \
         libsnappy-dev \
         libsqlite3-dev \
         libssh-dev \

ci/docker/ubuntu-24.04-cpp.dockerfile

@@ -94,6 +94,7 @@ RUN apt-get update -y -q && \
         libradospp-dev \
         libre2-dev \
         librtmp-dev \
+        libsimdjson-dev \
         libsnappy-dev \
         libsqlite3-dev \
         libssh-dev \

ci/scripts/cpp_build.sh

@@ -280,6 +280,7 @@ else
     -DProtobuf_SOURCE=${Protobuf_SOURCE:-} \
     -DRapidJSON_SOURCE=${RapidJSON_SOURCE:-} \
     -Dre2_SOURCE=${re2_SOURCE:-} \
+    -DSIMDJSON_SOURCE=${SIMDJSON_SOURCE:-} \
     -DSnappy_SOURCE=${Snappy_SOURCE:-} \
     -DThrift_SOURCE=${Thrift_SOURCE:-} \
     -Dutf8proc_SOURCE=${utf8proc_SOURCE:-} \

cpp/cmake_modules/ThirdpartyToolchain.cmake

@@ -64,6 +64,7 @@ set(ARROW_THIRDPARTY_DEPENDENCIES
     re2
     Protobuf
     RapidJSON
+    simdjson
     Snappy
     Substrait
     Thrift
@@ -211,6 +212,8 @@ macro(build_dependency DEPENDENCY_NAME)
     build_rapidjson()
   elseif("${DEPENDENCY_NAME}" STREQUAL "re2")
     build_re2()
+  elseif("${DEPENDENCY_NAME}" STREQUAL "simdjson")
+    build_simdjson()
   elseif("${DEPENDENCY_NAME}" STREQUAL "Snappy")
     build_snappy()
   elseif("${DEPENDENCY_NAME}" STREQUAL "Substrait")
@@ -381,6 +384,7 @@ endif()
 
 if(ARROW_PARQUET)
   set(ARROW_WITH_RAPIDJSON ON)
+  set(ARROW_WITH_SIMDJSON ON)
   set(ARROW_WITH_THRIFT ON)
 endif()
 
@@ -409,6 +413,7 @@ endif()
 
 if(ARROW_JSON OR ARROW_FLIGHT_SQL_ODBC)
   set(ARROW_WITH_RAPIDJSON ON)
+  set(ARROW_WITH_SIMDJSON ON)
 endif()
 
 if(ARROW_ORC OR ARROW_FLIGHT)
@@ -780,6 +785,14 @@ else()
            "https://github.com/aws/s2n-tls/archive/${ARROW_S2N_TLS_BUILD_VERSION}.tar.gz")
 endif()
 
+if(DEFINED ENV{ARROW_SIMDJSON_URL})
+  set(SIMDJSON_SOURCE_URL "$ENV{ARROW_SIMDJSON_URL}")
+else()
+  set_urls(SIMDJSON_SOURCE_URL
+          "https://github.com/simdjson/simdjson/archive/${ARROW_SIMDJSON_BUILD_VERSION}.tar.gz"
+          "${THIRDPARTY_MIRROR_URL}/simdjson-${ARROW_SIMDJSON_BUILD_VERSION}.tar.gz")
+endif()
+
 if(DEFINED ENV{ARROW_SNAPPY_URL})
   set(SNAPPY_SOURCE_URL "$ENV{ARROW_SNAPPY_URL}")
 else()
@@ -2613,6 +2626,45 @@ if(ARROW_WITH_RAPIDJSON)
                      FALSE)
 endif()
 
+macro(build_simdjson)
+  message(STATUS "Building simdjson from source")
+  set(SIMDJSON_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/simdjson_ep/src/simdjson_ep-install")
+  set(SIMDJSON_CMAKE_ARGS
+      ${EP_COMMON_CMAKE_ARGS}
+      -DSIMDJSON_JUST_LIBRARY=ON
+      -DSIMDJSON_BUILD_STATIC_LIB=ON
+      "-DCMAKE_INSTALL_PREFIX=${SIMDJSON_PREFIX}")
+
+  externalproject_add(simdjson_ep
+                      ${EP_COMMON_OPTIONS}
+                      PREFIX "${CMAKE_BINARY_DIR}"
+                      URL ${SIMDJSON_SOURCE_URL}
+                      URL_HASH "SHA256=${ARROW_SIMDJSON_BUILD_SHA256_CHECKSUM}"
+                      CMAKE_ARGS ${SIMDJSON_CMAKE_ARGS})
+
+  set(SIMDJSON_INCLUDE_DIR "${SIMDJSON_PREFIX}/include")
+  set(SIMDJSON_STATIC_LIB
+      "${SIMDJSON_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}simdjson${CMAKE_STATIC_LIBRARY_SUFFIX}")
+
+  # The include directory must exist before it is referenced by a target.
+  file(MAKE_DIRECTORY "${SIMDJSON_INCLUDE_DIR}")
+
+  add_library(simdjson::simdjson STATIC IMPORTED)
+  set_target_properties(simdjson::simdjson
+                        PROPERTIES IMPORTED_LOCATION "${SIMDJSON_STATIC_LIB}"
+                                   INTERFACE_INCLUDE_DIRECTORIES "${SIMDJSON_INCLUDE_DIR}")
+  add_dependencies(simdjson::simdjson simdjson_ep)
+
+  set(SIMDJSON_VENDORED TRUE)
+endmacro()
+
+if(ARROW_WITH_SIMDJSON)
+  resolve_dependency(simdjson
+                     USE_CONFIG
+                     IS_RUNTIME_DEPENDENCY
+                     FALSE)
+endif()
+
 macro(build_xsimd)
   message(STATUS "Building xsimd from source")
   set(XSIMD_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/xsimd_ep/src/xsimd_ep-install")

cpp/src/arrow/CMakeLists.txt

@@ -622,6 +622,11 @@ if(ARROW_WITH_RAPIDJSON)
     target_link_libraries(${ARROW_UTIL_TARGET} PRIVATE RapidJSON)
   endforeach()
 endif()
+if(ARROW_WITH_SIMDJSON)
+  foreach(ARROW_UTIL_TARGET ${ARROW_UTIL_TARGETS})
+    target_link_libraries(${ARROW_UTIL_TARGET} PRIVATE simdjson::simdjson)
+  endforeach()
+endif()
 if(ARROW_WITH_ZLIB)
   foreach(ARROW_UTIL_TARGET ${ARROW_UTIL_TARGETS})
     target_link_libraries(${ARROW_UTIL_TARGET} PRIVATE ZLIB::ZLIB)
@@ -1003,6 +1008,7 @@ if(ARROW_JSON)
                            json/reader.cc)
   foreach(ARROW_JSON_TARGET ${ARROW_JSON_TARGETS})
     target_link_libraries(${ARROW_JSON_TARGET} PRIVATE RapidJSON)
+    target_link_libraries(${ARROW_JSON_TARGET} PRIVATE simdjson::simdjson)
   endforeach()
 else()
   set(ARROW_JSON_TARGET_SHARED)

cpp/src/arrow/chunked_array_test.cc

@@ -171,7 +171,7 @@ TEST_F(TestChunkedArrayEqualsSameAddress, NestedTypeWithoutFloat) {
 }
 
 TEST_F(TestChunkedArrayEqualsSameAddress, FloatType) {
-  auto float64_array = ArrayFromJSON(float64(), "[0.0, 1.0, 2.0, NaN]");
+  auto float64_array = ArrayFromJSON(float64(), "[0.0, 1.0, 2.0, \"NaN\"]");
   ASSERT_OK_AND_ASSIGN(auto chunked_array, ChunkedArray::Make({float64_array}));
 
   ASSERT_FALSE(chunked_array->Equals(chunked_array));
@@ -182,7 +182,7 @@ TEST_F(TestChunkedArrayEqualsSameAddress, FloatType) {
 }
 
 TEST_F(TestChunkedArrayEqualsSameAddress, NestedTypeWithFloat) {
-  auto float64_array = ArrayFromJSON(float64(), "[0.0, 1.0, NaN]");
+  auto float64_array = ArrayFromJSON(float64(), "[0.0, 1.0, \"NaN\"]");
   ASSERT_OK_AND_ASSIGN(auto struct_array,
                        StructArray::Make({float64_array}, {"Float64Type"}));
   ASSERT_OK_AND_ASSIGN(auto chunked_array, ChunkedArray::Make({struct_array}));

cpp/src/arrow/compute/kernels/vector_cumulative_ops_test.cc

@@ -930,24 +930,24 @@ TEST(TestCumulativeMean, ConvenienceFunction) {
 
 TEST(TestCumulative, NaN) {
   // addition with NaN is always NaN
-  CheckVectorUnary("cumulative_sum", ArrayFromJSON(float64(), "[1, 2, NaN, 4, 5]"),
-                   ArrayFromJSON(float64(), "[1, 3, NaN, NaN, NaN]"));
+  CheckVectorUnary("cumulative_sum", ArrayFromJSON(float64(), "[1, 2, \"NaN\", 4, 5]"),
+                   ArrayFromJSON(float64(), R"([1, 3, "NaN", "NaN", "NaN"])"));
 
   // multiply with Nan is always NaN
-  CheckVectorUnary("cumulative_prod", ArrayFromJSON(float64(), "[1, 2, NaN, 4, 5]"),
-                   ArrayFromJSON(float64(), "[1, 2, NaN, NaN, NaN]"));
+  CheckVectorUnary("cumulative_prod", ArrayFromJSON(float64(), "[1, 2, \"NaN\", 4, 5]"),
+                   ArrayFromJSON(float64(), R"([1, 2, "NaN", "NaN", "NaN"])"));
 
   // max with NaN is always ignored because Nan > a always returns false
-  CheckVectorUnary("cumulative_max", ArrayFromJSON(float64(), "[1, 2, NaN, 4, 5]"),
+  CheckVectorUnary("cumulative_max", ArrayFromJSON(float64(), "[1, 2, \"NaN\", 4, 5]"),
                    ArrayFromJSON(float64(), "[1, 2, 2, 4, 5]"));
 
   // min with NaN is always ignored because Nan < a always returns false
-  CheckVectorUnary("cumulative_min", ArrayFromJSON(float64(), "[5, 4, NaN, 2, 1]"),
+  CheckVectorUnary("cumulative_min", ArrayFromJSON(float64(), "[5, 4, \"NaN\", 2, 1]"),
                    ArrayFromJSON(float64(), "[5, 4, 4, 2, 1]"));
 
   // mean with NaN is always Nan
-  CheckVectorUnary("cumulative_mean", ArrayFromJSON(float64(), "[5, 4, NaN, 2, 1]"),
-                   ArrayFromJSON(float64(), "[5, 4.5, NaN, NaN, NaN]"));
+  CheckVectorUnary("cumulative_mean", ArrayFromJSON(float64(), "[5, 4, \"NaN\", 2, 1]"),
+                   ArrayFromJSON(float64(), R"([5, 4.5, "NaN", "NaN", "NaN"])"));
 }
 }  // namespace compute
 }  // namespace arrow