Merge pull request #4 from NimbleEdge/jpuneet/call-tokenizer-is_loaded

Call tokenizer_->is_loaded

Author: nrjpoddar

.gitmodules

@@ -30,7 +30,7 @@
 	url = https://github.com/Maratyszcza/pthreadpool.git
 [submodule "extension/llm/tokenizers"]
 	path = extension/llm/tokenizers
-	url = https://github.com/pytorch-labs/tokenizers.git
+	url = https://github.com/NimbleEdge/tokenizers.git
 [submodule "kernels/optimized/third-party/eigen"]
 	path = kernels/optimized/third-party/eigen
 	url = https://gitlab.com/libeigen/eigen.git

extension/llm/runner/text_token_generator.h

@@ -151,7 +151,7 @@ class ET_EXPERIMENTAL TextTokenGenerator {
    */
   bool inline is_loaded() const {
     // Implementation to check if resources are loaded
-    return text_decoder_runner_->is_method_loaded();
+    return tokenizer_->is_loaded() && text_decoder_runner_->is_method_loaded();
   }
 
  private: