From 6284fb22b5be09de3dce905e70220cf1f850ed8b Mon Sep 17 00:00:00 2001 From: drownfish19 Date: Mon, 26 Aug 2024 09:22:09 +0000 Subject: [PATCH 1/2] fix --- paddlenlp/transformers/tokenizer_utils.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/paddlenlp/transformers/tokenizer_utils.py b/paddlenlp/transformers/tokenizer_utils.py index 4870a3e9b62a..8bdb120e2f3a 100644 --- a/paddlenlp/transformers/tokenizer_utils.py +++ b/paddlenlp/transformers/tokenizer_utils.py @@ -1875,8 +1875,12 @@ def decode_token( """tokenizer decoding for the streaming generation use case. This method can be overrided for tokenizer that doesn't follow this API""" # The prefix text is necessary only to defeat cleanup algorithms in the decode # which decide to add a space or not depending on the surrounding ids. - prefix_text = self.decode(all_input_ids[prefix_offset:read_offset], skip_special_tokens=False) - new_text = self.decode(all_input_ids[prefix_offset:], skip_special_tokens=False) + prefix_text = self.decode( + all_input_ids[prefix_offset:read_offset], skip_special_tokens=False, clean_up_tokenization_spaces=False + ) + new_text = self.decode( + all_input_ids[prefix_offset:], skip_special_tokens=False, clean_up_tokenization_spaces=False + ) if len(new_text) > len(prefix_text) and not new_text.endswith("�"): # utf-8 char at the end means it's a potential unfinished byte sequence From 0cd1c520d94794156eda7ca0e9d49890e03f5a36 Mon Sep 17 00:00:00 2001 From: drownfish19 Date: Wed, 28 Aug 2024 12:44:39 +0000 Subject: [PATCH 2/2] fix --- paddlenlp/transformers/tokenizer_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddlenlp/transformers/tokenizer_utils.py b/paddlenlp/transformers/tokenizer_utils.py index 8bdb120e2f3a..e7ba4f687577 100644 --- a/paddlenlp/transformers/tokenizer_utils.py +++ b/paddlenlp/transformers/tokenizer_utils.py @@ -1882,7 +1882,7 @@ def decode_token( all_input_ids[prefix_offset:], skip_special_tokens=False, clean_up_tokenization_spaces=False ) - if len(new_text) > len(prefix_text) and not new_text.endswith("�"): + if len(new_text) > len(prefix_text) and not prefix_text.endswith("�") and not new_text.endswith("�"): # utf-8 char at the end means it's a potential unfinished byte sequence # from byte fallback tokenization. # If it's in the middle, it's probably a real invalid id generated