Merge pull request PaddlePaddle#56 from smallv0221/yxp0301

Fix tokenizer_util padding
wangxicoding · Mar 2, 2021 · f4d4eac · f4d4eac
2 parents 054c95e + f32192c
commit f4d4eac
Showing 1 changed file with 4 additions and 4 deletions.
diff --git a/paddlenlp/transformers/tokenizer_utils.py b/paddlenlp/transformers/tokenizer_utils.py
@@ -766,9 +766,9 @@ def get_input_ids(text):
                     encoded_inputs["attention_mask"] = [1] * len(encoded_inputs[
                         "input_ids"]) + [0] * difference
                 if return_token_type_ids:
-                    # 0 for padding token mask
                     encoded_inputs["token_type_ids"] = (
-                        encoded_inputs["token_type_ids"] + [0] * difference)
+                        encoded_inputs["token_type_ids"] +
+                        [self.pad_token_type_id] * difference)
                 if return_special_tokens_mask:
                     encoded_inputs["special_tokens_mask"] = encoded_inputs[
                         "special_tokens_mask"] + [1] * difference
@@ -780,9 +780,9 @@ def get_input_ids(text):
                         1
                     ] * len(encoded_inputs["input_ids"])
                 if return_token_type_ids:
-                    # 0 for padding token mask
                     encoded_inputs["token_type_ids"] = (
-                        [0] * difference + encoded_inputs["token_type_ids"])
+                        [self.pad_token_type_id] * difference +
+                        encoded_inputs["token_type_ids"])
                 if return_special_tokens_mask:
                     encoded_inputs["special_tokens_mask"] = [
                         1