GH-49817: [Gandiva] Round long strings before decimal conversion

Signed-off-by: 1fanwang <1fannnw@gmail.com>

Author: 1fanwang

cpp/src/arrow/util/decimal.cc

@@ -801,14 +801,6 @@ static inline bool ShiftAndAddWithOverflow(std::string_view input, uint64_t out[
 
 namespace {
 
-struct DecimalComponents {
-  std::string_view whole_digits;
-  std::string_view fractional_digits;
-  int32_t exponent = 0;
-  char sign = 0;
-  bool has_exponent = false;
-};
-
 inline bool IsSign(char c) { return c == '-' || c == '+'; }
 
 inline bool IsDot(char c) { return c == '.'; }
@@ -829,7 +821,10 @@ inline size_t ParseDigitsRun(const char* s, size_t start, size_t size,
   return pos;
 }
 
-bool ParseDecimalComponents(const char* s, size_t size, DecimalComponents* out) {
+}  // namespace
+
+bool internal::ParseDecimalComponents(const char* s, size_t size,
+                                      DecimalComponents* out) {
   size_t pos = 0;
 
   if (size == 0) {
@@ -871,15 +866,17 @@ bool ParseDecimalComponents(const char* s, size_t size, DecimalComponents* out)
   return pos == size;
 }
 
+namespace {
+
 template <typename Decimal>
 Status DecimalFromString(const char* type_name, std::string_view s, Decimal* out,
                          int32_t* precision, int32_t* scale) {
   if (s.empty()) {
     return Status::Invalid("Empty string cannot be converted to ", type_name);
   }
 
-  DecimalComponents dec;
-  if (!ParseDecimalComponents(s.data(), s.size(), &dec)) {
+  internal::DecimalComponents dec;
+  if (!internal::ParseDecimalComponents(s.data(), s.size(), &dec)) {
     return Status::Invalid("The string '", s, "' is not a valid ", type_name, " number");
   }
 
@@ -953,8 +950,8 @@ Status SimpleDecimalFromString(const char* type_name, std::string_view s,
     return Status::Invalid("Empty string cannot be converted to ", type_name);
   }
 
-  DecimalComponents dec;
-  if (!ParseDecimalComponents(s.data(), s.size(), &dec)) {
+  internal::DecimalComponents dec;
+  if (!internal::ParseDecimalComponents(s.data(), s.size(), &dec)) {
     return Status::Invalid("The string '", s, "' is not a valid ", type_name, " number");
   }
 

cpp/src/arrow/util/decimal_internal.h

@@ -20,6 +20,7 @@
 #include <array>
 #include <cstdint>
 #include <limits>
+#include <string_view>
 #include <type_traits>
 
 #include "arrow/type_fwd.h"
@@ -30,6 +31,21 @@
 
 namespace arrow {
 
+namespace internal {
+
+struct DecimalComponents {
+  std::string_view whole_digits;
+  std::string_view fractional_digits;
+  int32_t exponent = 0;
+  char sign = 0;
+  bool has_exponent = false;
+};
+
+ARROW_EXPORT bool ParseDecimalComponents(const char* s, size_t size,
+                                         DecimalComponents* out);
+
+}  // namespace internal
+
 constexpr auto kInt32DecimalDigits =
     static_cast<size_t>(std::numeric_limits<int32_t>::digits10);
 

cpp/src/gandiva/gdv_function_stubs.cc

@@ -23,10 +23,12 @@
 #include <cstdio>
 #include <sstream>
 #include <string>
+#include <string_view>
 #include <vector>
 
 #include "arrow/util/base64.h"
 #include "arrow/util/bit_util.h"
+#include "arrow/util/decimal_internal.h"
 #include "arrow/util/double_conversion_internal.h"
 #include "arrow/util/value_parsing.h"
 
@@ -204,11 +206,51 @@ CRC_FUNCTION(utf8)
 CRC_FUNCTION(binary)
 
 int32_t gdv_fn_dec_from_string(int64_t context, const char* in, int32_t in_length,
-                               int32_t* precision_from_str, int32_t* scale_from_str,
-                               int64_t* dec_high_from_str, uint64_t* dec_low_from_str) {
+                               int32_t out_scale, int32_t* precision_from_str,
+                               int32_t* scale_from_str, int64_t* dec_high_from_str,
+                               uint64_t* dec_low_from_str) {
   arrow::Decimal128 dec;
-  auto status = arrow::Decimal128::FromString(std::string(in, in_length), &dec,
-                                              precision_from_str, scale_from_str);
+  const std::string_view input(in, in_length);
+  auto status =
+      arrow::Decimal128::FromString(input, &dec, precision_from_str, scale_from_str);
+  if (!status.ok() ||
+      static_cast<int64_t>(*scale_from_str) - out_scale > arrow::Decimal128::kMaxScale) {
+    arrow::internal::DecimalComponents components;
+    if (arrow::internal::ParseDecimalComponents(input.data(), input.size(),
+                                                &components)) {
+      std::string digits(components.whole_digits);
+      digits.append(components.fractional_digits);
+      digits.erase(0, digits.find_first_not_of('0'));
+      const int64_t num_digits =
+          static_cast<int64_t>(digits.size()) + out_scale -
+          static_cast<int64_t>(components.fractional_digits.size()) + components.exponent;
+      bool round_up = false;
+      if (num_digits < 0 || num_digits > arrow::Decimal128::kMaxPrecision) {
+        digits = "0";
+      } else {
+        round_up = num_digits < static_cast<int64_t>(digits.size()) &&
+                   digits[static_cast<size_t>(num_digits)] >= '5';
+        digits.resize(static_cast<size_t>(num_digits), '0');
+        if (digits.empty()) {
+          digits = "0";
+        }
+      }
+      status =
+          arrow::Decimal128::FromString(digits, &dec, precision_from_str, scale_from_str);
+      if (status.ok()) {
+        if (round_up) {
+          dec += arrow::Decimal128(1);
+          if (dec == arrow::Decimal128::GetScaleMultiplier(*precision_from_str)) {
+            ++(*precision_from_str);
+          }
+        }
+        if (components.sign == '-') {
+          dec.Negate();
+        }
+        *scale_from_str = out_scale;
+      }
+    }
+  }
   if (!status.ok()) {
     gdv_fn_context_set_error_msg(context, status.message().data());
     return -1;
@@ -948,6 +990,7 @@ arrow::Status ExportedStubFunctions::AddMappings(Engine* engine) const {
       types->i64_type(),      // context
       types->i8_ptr_type(),   // const char* in
       types->i32_type(),      // int32_t in_length
+      types->i32_type(),      // int32_t out_scale
       types->i32_ptr_type(),  // int32_t* precision_from_str
       types->i32_ptr_type(),  // int32_t* scale_from_str
       types->i64_ptr_type(),  // int64_t* dec_high_from_str

cpp/src/gandiva/gdv_function_stubs.h

@@ -128,8 +128,9 @@ const char* gdv_fn_sha1_decimal128(int64_t context, int64_t x_high, uint64_t x_l
                                    gdv_boolean x_isvalid, int32_t* out_length);
 
 int32_t gdv_fn_dec_from_string(int64_t context, const char* in, int32_t in_length,
-                               int32_t* precision_from_str, int32_t* scale_from_str,
-                               int64_t* dec_high_from_str, uint64_t* dec_low_from_str);
+                               int32_t out_scale, int32_t* precision_from_str,
+                               int32_t* scale_from_str, int64_t* dec_high_from_str,
+                               uint64_t* dec_low_from_str);
 
 char* gdv_fn_dec_to_string(int64_t context, int64_t x_high, uint64_t x_low,
                            int32_t x_scale, int32_t* dec_str_len);

cpp/src/gandiva/precompiled/decimal_wrapper.cc

@@ -403,8 +403,8 @@ void castDECIMAL_utf8(int64_t context, const char* in, int32_t in_length,
   int32_t precision_from_str;
   int32_t scale_from_str;
   int32_t status =
-      gdv_fn_dec_from_string(context, in, in_length, &precision_from_str, &scale_from_str,
-                             &dec_high_from_str, &dec_low_from_str);
+      gdv_fn_dec_from_string(context, in, in_length, out_scale, &precision_from_str,
+                             &scale_from_str, &dec_high_from_str, &dec_low_from_str);
   if (status != 0) {
     *out_high = 0;
     *out_low = 0;

cpp/src/gandiva/tests/decimal_test.cc

@@ -1083,18 +1083,60 @@ TEST_F(TestDecimal, TestCastDecimalVarCharInvalidInput) {
   // Create a row-batch with some sample data
   int num_records = 5;
 
-  // invalid input
-  auto invalid_in = MakeArrowArrayUtf8({"a10.5134", "-0.0", "-0.1", "10.516", "-1000"},
-                                       {true, false, true, true, true});
-
-  // prepare input record batch
-  auto in_batch_1 = arrow::RecordBatch::Make(schema, num_records, {invalid_in});
+  for (const std::string& invalid :
+       std::vector<std::string>{"a10.5134", "1." + std::string(100, '5') + "x",
+                                "1." + std::string(100, '5') + "e2147483648"}) {
+    SCOPED_TRACE(invalid);
+    auto invalid_in = MakeArrowArrayUtf8({invalid, "-0.0", "-0.1", "10.516", "-1000"},
+                                         {true, false, true, true, true});
+    auto in_batch = arrow::RecordBatch::Make(schema, num_records, {invalid_in});
+    arrow::ArrayVector outputs;
+    status = projector->Evaluate(*in_batch, pool_, &outputs);
+    EXPECT_FALSE(status.ok()) << status.message();
+    EXPECT_NE(status.message().find("not a valid decimal128 number"), std::string::npos);
+  }
+}
 
-  // Evaluate expression
-  arrow::ArrayVector outputs_1;
-  status = projector->Evaluate(*in_batch_1, pool_, &outputs_1);
-  EXPECT_FALSE(status.ok()) << status.message();
-  EXPECT_NE(status.message().find("not a valid decimal128 number"), std::string::npos);
+TEST_F(TestDecimal, TestCastDecimalVarCharLongInputs) {
+  struct TestCase {
+    std::string input;
+    int32_t precision;
+    int32_t scale;
+    std::string expected;
+  };
+  for (const auto& test_case : std::vector<TestCase>{
+           {"1." + std::string(50, '5'), 38, 37, "1." + std::string(36, '5') + "6"},
+           {"-1." + std::string(50, '5'), 38, 37, "-1." + std::string(36, '5') + "6"},
+           {"1." + std::string(100, '1'), 38, 37, "1." + std::string(37, '1')},
+           {"15." + std::string(50, '5') + "e-1", 38, 37,
+            "1." + std::string(36, '5') + "6"},
+           {"0." + std::string(37, '0') + "5" + std::string(70, '0'), 38, 37,
+            "0." + std::string(36, '0') + "1"},
+           {"0." + std::string(100, '0') + "5", 38, 37, "0"},
+           {"9." + std::string(80, '9'), 4, 2, "10.00"},
+           {"-9." + std::string(80, '9'), 4, 2, "-10.00"},
+           {"99." + std::string(80, '9'), 4, 2, "0.00"},
+           {"0001." + std::string(80, '0'), 4, 2, "1.00"},
+           {"1." + std::string(100, '0') + "e+2", 6, 2, "100.00"},
+           {"1234" + std::string(100, '0') + "e-102", 4, 2, "12.34"}}) {
+    SCOPED_TRACE(test_case.input);
+    auto decimal_type = arrow::decimal128(test_case.precision, test_case.scale);
+    auto field_str = field("in_str", utf8());
+    auto schema = arrow::schema({field_str});
+    auto expr = TreeExprBuilder::MakeExpression("castDECIMAL", {field_str},
+                                                field("out", decimal_type));
+    std::shared_ptr<Projector> projector;
+    ASSERT_OK(Projector::Make(schema, {expr}, TestConfiguration(), &projector));
+
+    auto input = MakeArrowArrayUtf8({test_case.input, ""}, {true, false});
+    auto batch = arrow::RecordBatch::Make(schema, input->length(), {input});
+    arrow::ArrayVector outputs;
+    ASSERT_OK(projector->Evaluate(*batch, pool_, &outputs));
+    auto expected = MakeArrowArrayDecimal(
+        decimal_type, MakeDecimalVector({test_case.expected, "0"}, test_case.scale),
+        {true, false});
+    EXPECT_ARROW_ARRAY_EQUALS(expected, outputs[0]);
+  }
 }
 
 TEST_F(TestDecimal, TestVarCharDecimalNestedCast) {