Use pytorch-labs/tokenizers and remove tokenizer/

Summary: Use our shiny new repo
https://github.com/pytorch-labs/tokenizers.

Test Plan: Rely on CI jobs

Reviewers:

Subscribers:

Tasks:

Tags:

.gitmodules

@@ -1,9 +1,3 @@
-[submodule "tokenizer/third-party/abseil-cpp"]
-	path = tokenizer/third-party/abseil-cpp
-	url = https://github.com/abseil/abseil-cpp.git
-[submodule "tokenizer/third-party/re2"]
-	path = tokenizer/third-party/re2
-	url = https://github.com/google/re2.git
-[submodule "tokenizer/third-party/sentencepiece"]
-	path = tokenizer/third-party/sentencepiece
-	url = https://github.com/google/sentencepiece.git
+[submodule "runner/third-party/tokenizers"]
+	path = runner/third-party/tokenizers
+	url = https://github.com/pytorch-labs/tokenizers

CMakeLists.txt

@@ -7,18 +7,21 @@ ELSE()
 ENDIF()
 
 project(Torchchat)
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-attributes")
 
 # include tokenizer
-add_subdirectory(tokenizer)
+add_subdirectory(runner/third-party/tokenizers)
 
 # include et_run executable
 include(runner/et.cmake)
 if(TARGET et_run)
-    target_link_libraries(et_run PUBLIC tokenizer microkernels-prod)
+    target_link_libraries(et_run PUBLIC tokenizers microkernels-prod)
+    target_include_directories(et_run PUBLIC runner/third-party/tokenizers/include)
 endif()
 
 # include aoti_run executable
 include(runner/aoti.cmake)
 if(TARGET aoti_run)
-    target_link_libraries(aoti_run tokenizer)
+    target_link_libraries(aoti_run tokenizers)
+    target_include_directories(aoti_run PUBLIC runner/third-party/tokenizers/include)
 endif()