TextPreprocesor 2: TextPreprocesor for BertTokenizer

Extracted out the text preprocessing logic from BertNLClassifier and merged it into the TextPreprocessor.

PiperOrigin-RevId: 410059736

Author: lu-wang-g

tensorflow_lite_support/cc/task/processor/BUILD

@@ -115,9 +115,11 @@ cc_library_with_tflite(
         "//tensorflow_lite_support/cc/task/core:task_utils",
         "//tensorflow_lite_support/cc/text/tokenizers:regex_tokenizer",
         "//tensorflow_lite_support/cc/text/tokenizers:tokenizer",
+        "//tensorflow_lite_support/cc/text/tokenizers:tokenizer_utils",
         "//tensorflow_lite_support/cc/utils:common_utils",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/status",
+        "@com_google_absl//absl/strings",
         "@com_google_absl//absl/strings:str_format",
     ],
 )

tensorflow_lite_support/cc/task/processor/processor.cc

@@ -14,6 +14,9 @@ limitations under the License.
 ==============================================================================*/
 #include "tensorflow_lite_support/cc/task/processor/processor.h"
 
+#include <iterator>
+#include <sstream>
+
 namespace tflite {
 namespace task {
 namespace processor {
@@ -55,6 +58,13 @@ absl::Status Processor::SanityCheck(int num_expected_tensors,
   return absl::OkStatus();
 }
 
+std::string Processor::GetTensorIndexString() {
+  std::stringstream stream;
+  std::copy(tensor_indices_.begin(), tensor_indices_.end(),
+            std::ostream_iterator<int>(stream, " "));
+  return stream.str();
+}
+
 }  // namespace processor
 }  // namespace task
 }  // namespace tflite

tensorflow_lite_support/cc/task/processor/processor.h

@@ -94,6 +94,9 @@ class Processor {
     return engine_->metadata_extractor();
   }
 
+  // Gets the tesnor indices in string format.
+  std::string GetTensorIndexString();
+
   core::TfLiteEngine* engine_;
   const std::vector<int> tensor_indices_;