Also ignore errors on input prompts

Mug · Mug · commit 5f81400fcb28 · 2023-04-26T14:45:51.000+02:00
diff --git a/examples/low_level_api/low_level_api_chat_cpp.py b/examples/low_level_api/low_level_api_chat_cpp.py
@@ -201,7 +201,7 @@ def __init__(self, params: GptParams) -> None:
 	# tokenize a prompt
 	def _tokenize(self, prompt, bos=True):
 		_arr = (llama_cpp.llama_token * (len(prompt) + 1))()
-		_n = llama_cpp.llama_tokenize(self.ctx, prompt.encode("utf8"), _arr, len(_arr), bos)
+		_n = llama_cpp.llama_tokenize(self.ctx, prompt.encode("utf8", errors="ignore"), _arr, len(_arr), bos)
 		return _arr[:_n]
 
 	def set_color(self, c):
diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py
@@ -358,7 +358,7 @@ def create_embedding(self, input: str) -> Embedding:
         if self.verbose:
             llama_cpp.llama_reset_timings(self.ctx)
 
-        tokens = self.tokenize(input.encode("utf-8"))
+        tokens = self.tokenize(input.encode("utf-8", errors="ignore"))
         self.reset()
         self.eval(tokens)
         n_tokens = len(tokens)
@@ -416,7 +416,7 @@ def _create_completion(
         completion_tokens: List[llama_cpp.llama_token] = []
         # Add blank space to start of prompt to match OG llama tokenizer
         prompt_tokens: List[llama_cpp.llama_token] = self.tokenize(
-            b" " + prompt.encode("utf-8")
+            b" " + prompt.encode("utf-8", errors="ignore")
         )
         text: bytes = b""
         returned_characters: int = 0
@@ -431,7 +431,7 @@ def _create_completion(
             )
 
         if stop != []:
-            stop_sequences = [s.encode("utf-8") for s in stop]
+            stop_sequences = [s.encode("utf-8", errors="ignore") for s in stop]
         else:
             stop_sequences = []
 
diff --git a/tests/test_llama.py b/tests/test_llama.py
@@ -24,7 +24,7 @@ def mock_eval(*args, **kwargs):
     monkeypatch.setattr("llama_cpp.llama_cpp.llama_eval", mock_eval)
 
     output_text = " jumps over the lazy dog."
-    output_tokens = llama.tokenize(output_text.encode("utf-8"))
+    output_tokens = llama.tokenize(output_text.encode("utf-8", errors="ignore"))
     token_eos = llama.token_eos()
     n = 0
 
