[Don't review yet, reworking] B&B CPU determinism

benchmarks/linear_programming/cuopt/run_mip.cpp

@@ -147,7 +147,9 @@ int run_single_file(std::string file_path,
                     int num_cpu_threads,
                     bool write_log_file,
                     bool log_to_console,
-                    double time_limit)
+                    double time_limit,
+                    double work_limit,
+                    bool deterministic)
 {
   const raft::handle_t handle_{};
   cuopt::linear_programming::mip_solver_settings_t<int, double> settings;
@@ -197,13 +199,16 @@ int run_single_file(std::string file_path,
     }
   }
 
-  settings.time_limit                    = time_limit;
-  settings.heuristics_only               = heuristics_only;
-  settings.num_cpu_threads               = num_cpu_threads;
-  settings.log_to_console                = log_to_console;
+  settings.time_limit       = time_limit;
+  settings.work_limit       = work_limit;
+  settings.heuristics_only  = heuristics_only;
+  settings.num_cpu_threads  = num_cpu_threads;
+  settings.log_to_console   = log_to_console;
+  settings.determinism_mode = deterministic ? CUOPT_MODE_DETERMINISTIC : CUOPT_MODE_OPPORTUNISTIC;
   settings.tolerances.relative_tolerance = 1e-12;
   settings.tolerances.absolute_tolerance = 1e-6;
   settings.presolve                      = true;
+  settings.seed                          = 42;
   cuopt::linear_programming::benchmark_info_t benchmark_info;
   settings.benchmark_info_ptr = &benchmark_info;
   auto start_run_solver       = std::chrono::high_resolution_clock::now();
@@ -256,7 +261,9 @@ void run_single_file_mp(std::string file_path,
                         int num_cpu_threads,
                         bool write_log_file,
                         bool log_to_console,
-                        double time_limit)
+                        double time_limit,
+                        double work_limit,
+                        bool deterministic)
 {
   std::cout << "running file " << file_path << " on gpu : " << device << std::endl;
   auto memory_resource = make_async();
@@ -271,7 +278,9 @@ void run_single_file_mp(std::string file_path,
                                   num_cpu_threads,
                                   write_log_file,
                                   log_to_console,
-                                  time_limit);
+                                  time_limit,
+                                  work_limit,
+                                  deterministic);
   // this is a bad design to communicate the result but better than adding complexity of IPC or
   // pipes
   exit(sol_found);
@@ -341,7 +350,12 @@ int main(int argc, char* argv[])
     .default_value(std::string("t"));
 
   program.add_argument("--time-limit")
-    .help("time limit")
+    .help("time limit in seconds")
+    .scan<'g', double>()
+    .default_value(std::numeric_limits<double>::infinity());
+
+  program.add_argument("--work-limit")
+    .help("work unit limit (for deterministic mode)")
     .scan<'g', double>()
     .default_value(std::numeric_limits<double>::infinity());
 
@@ -354,6 +368,11 @@ int main(int argc, char* argv[])
     .help("track allocations (t/f)")
     .default_value(std::string("f"));
 
+  program.add_argument("-d", "--determinism")
+    .help("enable deterministic mode")
+    .default_value(false)
+    .implicit_value(true);
+
   // Parse arguments
   try {
     program.parse_args(argc, argv);
@@ -368,6 +387,7 @@ int main(int argc, char* argv[])
   std::string run_dir_arg = program.get<std::string>("--run-dir");
   bool run_dir            = run_dir_arg[0] == 't';
   double time_limit       = program.get<double>("--time-limit");
+  double work_limit       = program.get<double>("--work-limit");
 
   bool run_selected = program.get<std::string>("--run-selected")[0] == 't';
   int n_gpus        = program.get<int>("--n-gpus");
@@ -382,6 +402,7 @@ int main(int argc, char* argv[])
   bool log_to_console    = program.get<std::string>("--log-to-console")[0] == 't';
   double memory_limit    = program.get<double>("--memory-limit");
   bool track_allocations = program.get<std::string>("--track-allocations")[0] == 't';
+  bool deterministic     = program.get<bool>("--determinism");
 
   if (num_cpu_threads < 0) { num_cpu_threads = omp_get_max_threads() / n_gpus; }
 
@@ -469,7 +490,9 @@ int main(int argc, char* argv[])
                                num_cpu_threads,
                                write_log_file,
                                log_to_console,
-                               time_limit);
+                               time_limit,
+                               work_limit,
+                               deterministic);
           } else if (sys_pid < 0) {
             std::cerr << "Fork failed!" << std::endl;
             exit(1);
@@ -509,7 +532,9 @@ int main(int argc, char* argv[])
                     num_cpu_threads,
                     write_log_file,
                     log_to_console,
-                    time_limit);
+                    time_limit,
+                    work_limit,
+                    deterministic);
   }
 
   return 0;

cpp/CMakeLists.txt

@@ -491,9 +491,9 @@ target_link_libraries(cuopt_cli
   PRIVATE
 )
   # Use RUNPATH when building locally in order to allow LD_LIBRARY_PATH to override the conda env path
-if(NOT DEFINED INSTALL_TARGET OR "${INSTALL_TARGET}" STREQUAL "")
-  target_link_options(cuopt_cli PRIVATE -Wl,--enable-new-dtags)
-endif()
+# if(NOT DEFINED INSTALL_TARGET OR "${INSTALL_TARGET}" STREQUAL "")
+#   target_link_options(cuopt_cli PRIVATE -Wl,--enable-new-dtags)
+# endif()
 set_property(TARGET cuopt_cli PROPERTY INSTALL_RPATH "$ORIGIN/../${lib_dir}")
 
 # adds the cuopt_cli executable to the runtime deb package
@@ -525,9 +525,9 @@ if(BUILD_MIP_BENCHMARKS AND NOT BUILD_LP_ONLY)
     OpenMP::OpenMP_CXX
     PRIVATE
   )
-  if(NOT DEFINED INSTALL_TARGET OR "${INSTALL_TARGET}" STREQUAL "")
-    target_link_options(solve_MIP PRIVATE -Wl,--enable-new-dtags)
-  endif()
+  # if(NOT DEFINED INSTALL_TARGET OR "${INSTALL_TARGET}" STREQUAL "")
+  #   target_link_options(solve_MIP PRIVATE -Wl,--enable-new-dtags)
+  # endif()
 
   target_include_directories(solve_MIP
     PRIVATE
@@ -559,9 +559,9 @@ if(BUILD_LP_BENCHMARKS)
     OpenMP::OpenMP_CXX
     PRIVATE
   )
-  if(NOT DEFINED INSTALL_TARGET OR "${INSTALL_TARGET}" STREQUAL "")
-    target_link_options(solve_LP PRIVATE -Wl,--enable-new-dtags)
-  endif()
+  # if(NOT DEFINED INSTALL_TARGET OR "${INSTALL_TARGET}" STREQUAL "")
+  #   target_link_options(solve_LP PRIVATE -Wl,--enable-new-dtags)
+  # endif()
 endif()
 
 

cpp/include/cuopt/linear_programming/constants.h

@@ -1,6 +1,6 @@
 /* clang-format off */
 /*
- * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: Apache-2.0
  */
 /* clang-format on */
@@ -32,6 +32,7 @@
 #define CUOPT_DUAL_INFEASIBLE_TOLERANCE   "dual_infeasible_tolerance"
 #define CUOPT_ITERATION_LIMIT             "iteration_limit"
 #define CUOPT_TIME_LIMIT                  "time_limit"
+#define CUOPT_WORK_LIMIT                  "work_limit"
 #define CUOPT_PDLP_SOLVER_MODE            "pdlp_solver_mode"
 #define CUOPT_METHOD                      "method"
 #define CUOPT_PER_CONSTRAINT_RESIDUAL     "per_constraint_residual"
@@ -49,6 +50,7 @@
 #define CUOPT_CUDSS_DETERMINISTIC         "cudss_deterministic"
 #define CUOPT_PRESOLVE                    "presolve"
 #define CUOPT_DUAL_POSTSOLVE              "dual_postsolve"
+#define CUOPT_MIP_DETERMINISM_MODE        "mip_determinism_mode"
 #define CUOPT_MIP_ABSOLUTE_TOLERANCE      "mip_absolute_tolerance"
 #define CUOPT_MIP_RELATIVE_TOLERANCE      "mip_relative_tolerance"
 #define CUOPT_MIP_INTEGRALITY_TOLERANCE   "mip_integrality_tolerance"
@@ -57,11 +59,16 @@
 #define CUOPT_MIP_HEURISTICS_ONLY         "mip_heuristics_only"
 #define CUOPT_MIP_SCALING                 "mip_scaling"
 #define CUOPT_MIP_PRESOLVE                "mip_presolve"
+#define CUOPT_MIP_SEED                    "mip_seed"
 #define CUOPT_SOLUTION_FILE               "solution_file"
 #define CUOPT_NUM_CPU_THREADS             "num_cpu_threads"
 #define CUOPT_NUM_GPUS                    "num_gpus"
 #define CUOPT_USER_PROBLEM_FILE           "user_problem_file"
 
+/* @brief MIP determinism mode constants */
+#define CUOPT_MODE_OPPORTUNISTIC 0
+#define CUOPT_MODE_DETERMINISTIC 1
+
 /* @brief LP/MIP termination status constants */
 #define CUOPT_TERIMINATION_STATUS_NO_TERMINATION   0
 #define CUOPT_TERIMINATION_STATUS_OPTIMAL          1
@@ -73,6 +80,7 @@
 #define CUOPT_TERIMINATION_STATUS_PRIMAL_FEASIBLE  7
 #define CUOPT_TERIMINATION_STATUS_FEASIBLE_FOUND   8
 #define CUOPT_TERIMINATION_STATUS_CONCURRENT_LIMIT 9
+#define CUOPT_TERIMINATION_STATUS_WORK_LIMIT       10
 
 /* @brief The objective sense constants */
 #define CUOPT_MINIMIZE 1

cpp/include/cuopt/linear_programming/mip/solver_settings.hpp

@@ -1,6 +1,6 @@
 /* clang-format off */
 /*
- * SPDX-FileCopyrightText: Copyright (c) 2023-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: Apache-2.0
  */
 /* clang-format on */
@@ -79,6 +79,7 @@ class mip_solver_settings_t {
   tolerances_t tolerances;
 
   f_t time_limit       = std::numeric_limits<f_t>::infinity();
+  f_t work_limit       = std::numeric_limits<f_t>::infinity();
   bool heuristics_only = false;
   i_t num_cpu_threads  = -1;  // -1 means use default number of threads in branch and bound
   i_t num_gpus         = 1;
@@ -91,6 +92,23 @@ class mip_solver_settings_t {
   std::vector<std::shared_ptr<rmm::device_uvector<f_t>>> initial_solutions;
   bool mip_scaling = true;
   bool presolve    = true;
+  /**
+   * @brief Determinism mode for MIP solver.
+   *
+   * Controls the determinism behavior of the MIP solver:
+   * - CUOPT_MODE_OPPORTUNISTIC (0): Default mode, allows non-deterministic
+   *   parallelism for better performance
+   * - CUOPT_MODE_DETERMINISTIC (1): Ensures deterministic results across runs
+   *   at potential cost of performance
+   */
+  int determinism_mode = CUOPT_MODE_OPPORTUNISTIC;
+  /**
+   * @brief Random seed for the MIP solver.
+   *
+   * Controls the initial seed for random number generation in the solver.
+   * Use -1 to generate a random seed.
+   */
+  i_t seed = -1;
   // this is for extracting info from different places of the solver during
   // benchmarks
   benchmark_info_t* benchmark_info_ptr = nullptr;

cpp/include/cuopt/linear_programming/mip/solver_solution.hpp

@@ -1,6 +1,6 @@
 /* clang-format off */
 /*
- * SPDX-FileCopyrightText: Copyright (c) 2022-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: Apache-2.0
  */
 /* clang-format on */
@@ -30,6 +30,7 @@ enum class mip_termination_status_t : int8_t {
   Infeasible    = CUOPT_TERIMINATION_STATUS_INFEASIBLE,
   Unbounded     = CUOPT_TERIMINATION_STATUS_UNBOUNDED,
   TimeLimit     = CUOPT_TERIMINATION_STATUS_TIME_LIMIT,
+  WorkLimit     = CUOPT_TERIMINATION_STATUS_WORK_LIMIT,
 };
 
 template <typename i_t, typename f_t>

cpp/src/CMakeLists.txt

@@ -1,12 +1,24 @@
 # cmake-format: off
-# SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 # cmake-format: on
 
 set(UTIL_SRC_FILES ${CMAKE_CURRENT_SOURCE_DIR}/utilities/seed_generator.cu
                    ${CMAKE_CURRENT_SOURCE_DIR}/utilities/logger.cpp
                    ${CMAKE_CURRENT_SOURCE_DIR}/utilities/version_info.cpp
-                   ${CMAKE_CURRENT_SOURCE_DIR}/utilities/timestamp_utils.cpp)
+                   ${CMAKE_CURRENT_SOURCE_DIR}/utilities/timestamp_utils.cpp
+                   ${CMAKE_CURRENT_SOURCE_DIR}/utilities/work_unit_predictor.cpp
+                   ${CMAKE_CURRENT_SOURCE_DIR}/utilities/work_unit_scheduler.cpp
+                   ${CMAKE_CURRENT_SOURCE_DIR}/utilities/models/fj_predictor/main.cpp
+                   ${CMAKE_CURRENT_SOURCE_DIR}/utilities/models/fj_predictor/quantize.cpp
+                   ${CMAKE_CURRENT_SOURCE_DIR}/utilities/models/cpufj_predictor/main.cpp
+                   ${CMAKE_CURRENT_SOURCE_DIR}/utilities/models/cpufj_predictor/quantize.cpp
+                   ${CMAKE_CURRENT_SOURCE_DIR}/utilities/models/dualsimplex_predictor/main.cpp
+                   ${CMAKE_CURRENT_SOURCE_DIR}/utilities/models/dualsimplex_predictor/quantize.cpp
+                   ${CMAKE_CURRENT_SOURCE_DIR}/utilities/models/pdlp_predictor/main.cpp
+                   ${CMAKE_CURRENT_SOURCE_DIR}/utilities/models/pdlp_predictor/quantize.cpp
+                   ${CMAKE_CURRENT_SOURCE_DIR}/utilities/models/bounds_strengthening_predictor/main.cpp
+                   ${CMAKE_CURRENT_SOURCE_DIR}/utilities/models/bounds_strengthening_predictor/quantize.cpp)
 
 add_subdirectory(linear_programming)
 add_subdirectory(math_optimization)

cpp/src/dual_simplex/barrier.cu

@@ -1388,7 +1388,7 @@ class iteration_data_t {
     // v = alpha * A * w + beta * v = alpha * A * Dinv * A^T * y + beta * v
     matrix_vector_multiply(A, alpha, w, beta, v);
     if (debug) {
-      printf("||A|| = %.16e\n", vector_norm2<i_t, f_t>(A.x));
+      printf("||A|| = %.16e\n", vector_norm2<i_t, f_t>(A.x.underlying()));
       printf("||w|| = %.16e\n", vector_norm2<i_t, f_t>(w));
       printf("||v|| = %.16e\n", vector_norm2<i_t, f_t>(v));
     }

cpp/src/dual_simplex/basis_solves.cpp

@@ -12,9 +12,15 @@
 #include <dual_simplex/singletons.hpp>
 #include <dual_simplex/tic_toc.hpp>
 #include <dual_simplex/triangle_solve.hpp>
+#include <utilities/memory_instrumentation.hpp>
+
+#include <raft/common/nvtx.hpp>
 
 namespace cuopt::linear_programming::dual_simplex {
 
+// Import instrumented vector type
+using cuopt::ins_vector;
+
 template <typename i_t>
 i_t reorder_basic_list(const std::vector<i_t>& q, std::vector<i_t>& basic_list)
 {
@@ -57,14 +63,14 @@ void get_basis_from_vstatus(i_t m,
 
 namespace {
 
-template <typename i_t, typename f_t>
+template <typename i_t, typename f_t, typename VectorI>
 void write_singleton_info(i_t m,
                           i_t col_singletons,
                           i_t row_singletons,
                           const csc_matrix_t<i_t, f_t>& B,
-                          const std::vector<i_t>& row_perm,
-                          const std::vector<i_t>& row_perm_inv,
-                          const std::vector<i_t>& col_perm)
+                          const VectorI& row_perm,
+                          const VectorI& row_perm_inv,
+                          const VectorI& col_perm)
 {
   FILE* file = fopen("singleton_debug.m", "w");
   if (file != NULL) {
@@ -94,7 +100,7 @@ void write_singleton_info(i_t m,
   fclose(file);
 }
 
-template <typename i_t, typename f_t>
+template <typename i_t, typename f_t, typename VectorI>
 void write_factor_info(const char* filename,
                        i_t m,
                        i_t row_singletons,
@@ -104,8 +110,8 @@ void write_factor_info(const char* filename,
                        const csc_matrix_t<i_t, f_t>& D,
                        const csc_matrix_t<i_t, f_t>& L,
                        const csc_matrix_t<i_t, f_t>& U,
-                       const std::vector<i_t>& row_perm,
-                       const std::vector<i_t>& col_perm)
+                       const VectorI& row_perm,
+                       const VectorI& col_perm)
 {
   FILE* file = fopen(filename, "w");
   if (file != NULL) {
@@ -165,6 +171,7 @@ i_t factorize_basis(const csc_matrix_t<i_t, f_t>& A,
                     std::vector<i_t>& deficient,
                     std::vector<i_t>& slacks_needed)
 {
+  raft::common::nvtx::range scope("LU::factorize_basis");
   const i_t m              = basic_list.size();
   constexpr f_t medium_tol = 1e-12;
 
@@ -765,6 +772,8 @@ i_t b_transpose_solve(const csc_matrix_t<i_t, f_t>& L,
   // U'*r = c
   // L'*w = r
 
+  raft::common::nvtx::range scope("LU::b_transpose_solve");
+
   // Solve for r such that U'*r = c
   std::vector<f_t> r = rhs;
   upper_triangular_transpose_solve(U, r);