From 2f610e7af8a7e283c8c3b8fc8bb66e62fe0e9807 Mon Sep 17 00:00:00 2001
From: Hgtc <hgtcwl@foxmail.com>
Date: Tue, 20 Oct 2020 17:58:43 +0800
Subject: [PATCH 01/31] ADD: add whole word mask proxy for both eng and chinese

---
 examples/language-modeling/chinese_ref.py     | 143 ++++++++++++++++++
 .../run_language_modeling.py                  |  14 +-
 src/transformers/data/data_collator.py        |  95 ++++++++++--
 .../data/datasets/language_modeling.py        |  41 ++++-
 4 files changed, 277 insertions(+), 16 deletions(-)
 create mode 100644 examples/language-modeling/chinese_ref.py

diff --git a/examples/language-modeling/chinese_ref.py b/examples/language-modeling/chinese_ref.py
new file mode 100644
index 000000000000..e0bfbd1add05
--- /dev/null
+++ b/examples/language-modeling/chinese_ref.py
@@ -0,0 +1,143 @@
+import json
+import random
+import argparse
+from ltp import LTP
+from transformers.tokenization_bert import BertTokenizer
+
+from typing import List
+
+
+def _is_chinese_char(cp):
+    """Checks whether CP is the codepoint of a CJK character."""
+    # This defines a "chinese character" as anything in the CJK Unicode block:
+    #   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
+    #
+    # Note that the CJK Unicode block is NOT all Japanese and Korean characters,
+    # despite its name. The modern Korean Hangul alphabet is a different block,
+    # as is Japanese Hiragana and Katakana. Those alphabets are used to write
+    # space-separated words, so they are not treated specially and handled
+    # like the all of the other languages.
+    if (
+            (cp >= 0x4E00 and cp <= 0x9FFF)
+            or (cp >= 0x3400 and cp <= 0x4DBF)  #
+            or (cp >= 0x20000 and cp <= 0x2A6DF)  #
+            or (cp >= 0x2A700 and cp <= 0x2B73F)  #
+            or (cp >= 0x2B740 and cp <= 0x2B81F)  #
+            or (cp >= 0x2B820 and cp <= 0x2CEAF)  #
+            or (cp >= 0xF900 and cp <= 0xFAFF)
+            or (cp >= 0x2F800 and cp <= 0x2FA1F)  #
+    ):  #
+        return True
+
+    return False
+
+
+def is_chinese(word: str):
+    # word like '180' or '身高' or '神'
+    for char in word:
+        char = ord(char)
+        if not _is_chinese_char(char):
+            return 0
+    return 1
+
+
+def get_chinese_word(tokens: List[str]):
+    word_set = set()
+
+    for token in tokens:
+        chinese_word = len(token) > 1 and is_chinese(token)
+        if chinese_word:
+            word_set.add(token)
+    word_list = list(word_set)
+    return word_list
+
+
+def add_sub_symbol(bert_tokens: List[str], chinese_word_set: set()):
+    if not chinese_word_set:
+        return bert_tokens
+    max_word_len = max([len(w) for w in chinese_word_set])
+
+    bert_word = bert_tokens
+    start, end = 0, len(bert_word)
+    while start < end:
+        single_word = True
+        if is_chinese(bert_word[start]):
+            l = min(end - start, max_word_len)
+            for i in range(l, 1, -1):
+                whole_word = ''.join(bert_word[start:start + i])
+                if whole_word in chinese_word_set:
+                    for j in range(start + 1, start + i):
+                        bert_word[j] = '##' + bert_word[j]
+                    start = start + i
+                    single_word = False
+                    break
+        if single_word:
+            start += 1
+    return bert_word
+
+
+def prepare_ref(lines: List[str], ltp_tokenizer: LTP, bert_tokenizer: BertTokenizer):
+    ltp_res = []
+
+    for i in range(0, len(lines), 100):
+        res = ltp_tokenizer.seg(lines[i:i + 100])[0]
+        res = [get_chinese_word(r) for r in res]
+        ltp_res.extend(res)
+    assert len(ltp_res) == len(lines)
+
+    bert_res = []
+    for i in range(0, len(lines), 100):
+        res = bert_tokenizer(lines[i:i + 100], add_special_tokens=True, truncation=True, max_length=512)
+        bert_res.extend(res['input_ids'])
+    assert len(bert_res) == len(lines)
+
+    ref_ids = []
+    for input_ids, chinese_word in zip(bert_res, ltp_res):
+
+        input_tokens = []
+        for id in input_ids:
+            token = bert_tokenizer._convert_id_to_token(id)
+            input_tokens.append(token)
+        input_tokens = add_sub_symbol(input_tokens, chinese_word)
+        ref_id = []
+        # We only save pos of chinese subwords start with ##, which mean is part of a whole word.
+        for i, token in enumerate(input_tokens):
+            if token[:2] == '##':
+                clean_token = token[2:]
+                # save chinese tokens' pos
+                if len(clean_token) == 1 and _is_chinese_char(ord(clean_token)):
+                    ref_id.append(i)
+        ref_ids.append(ref_id)
+
+    assert len(ref_ids) == len(bert_res)
+
+    return ref_ids
+
+
+def main(args):
+    # For Chinese (Ro)Bert, the best result is from : RoBERTa-wwm-ext (https://github.com/ymcui/Chinese-BERT-wwm)
+    # If we want to fine-tune these model, we have to use same tokenizer : LTP (https://github.com/HIT-SCIR/ltp)
+    with open(args.file_name, 'r', encoding='utf-8') as f:
+        data = f.readlines()
+
+    ltp_tokenizer = LTP(args.ltp)  # faster in GPU device
+    bert_tokenizer = BertTokenizer.from_pretrained(args.bert)
+
+    ref_ids = prepare_ref(data, ltp_tokenizer, bert_tokenizer)
+
+    with open(args.save_path, 'w', encoding='utf-8') as f:
+        data = [json.dumps(ref) + '\n' for ref in ref_ids]
+        f.writelines(data)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='prepare_chinese_ref')
+    parser.add_argument('--file_name', type=str, default='./resources/chinese-demo.txt',
+                        help='file need process, same as training data in lm')
+    parser.add_argument('--ltp', type=str, default='./resources/ltp',
+                        help='resources for LTP tokenizer, usually a path')
+    parser.add_argument('--bert', type=str, default='./resources/robert', help='resources for Bert tokenizer')
+    parser.add_argument('--save_path', type=str, default='./resources/ref.txt', help='path to save res')
+
+    args = parser.parse_args()
+    main(args)
diff --git a/examples/language-modeling/run_language_modeling.py b/examples/language-modeling/run_language_modeling.py
index f3ce40e32107..f8ec07c9cad1 100644
--- a/examples/language-modeling/run_language_modeling.py
+++ b/examples/language-modeling/run_language_modeling.py
@@ -39,6 +39,7 @@
     DataCollatorForPermutationLanguageModeling,
     HfArgumentParser,
     LineByLineTextDataset,
+    LineByLineWithRefDataset,
     PreTrainedTokenizer,
     TextDataset,
     Trainer,
@@ -101,6 +102,10 @@ class DataTrainingArguments:
         default=None,
         metadata={"help": "An optional input evaluation data file to evaluate the perplexity on (a text file)."},
     )
+    chinese_ref_file: Optional[str] = field(
+        default=False,
+        metadata={"help": "An optional input ref data file for whole word mask(wwm) in Chinees."},
+    )
     line_by_line: bool = field(
         default=False,
         metadata={"help": "Whether distinct lines of text in the dataset are to be handled as distinct sequences."},
@@ -109,6 +114,9 @@ class DataTrainingArguments:
     mlm: bool = field(
         default=False, metadata={"help": "Train with masked-language modeling loss instead of language modeling."}
     )
+    wwm: bool = field(
+        default=False, metadata={"help": "Use Whole Word Mask."}
+    )
     mlm_probability: float = field(
         default=0.15, metadata={"help": "Ratio of tokens to mask for masked language modeling loss"}
     )
@@ -143,6 +151,10 @@ def get_dataset(
 ):
     def _dataset(file_path):
         if args.line_by_line:
+            if args.chinese_ref_file:
+                return LineByLineWithRefDataset(tokenizer=tokenizer, file_path=file_path, block_size=args.block_size,
+                                             ref_path=args.chinese_ref_file)
+
             return LineByLineTextDataset(tokenizer=tokenizer, file_path=file_path, block_size=args.block_size)
         else:
             return TextDataset(
@@ -271,7 +283,7 @@ def main():
         )
     else:
         data_collator = DataCollatorForLanguageModeling(
-            tokenizer=tokenizer, mlm=data_args.mlm, mlm_probability=data_args.mlm_probability
+            tokenizer=tokenizer, mlm=data_args.mlm, wwm=data_args.wwm, mlm_probability=data_args.mlm_probability
         )
 
     # Initialize our Trainer
diff --git a/src/transformers/data/data_collator.py b/src/transformers/data/data_collator.py
index f2fec6bb2e97..d8b6f687333f 100644
--- a/src/transformers/data/data_collator.py
+++ b/src/transformers/data/data_collator.py
@@ -1,6 +1,6 @@
 from dataclasses import dataclass
 from typing import Any, Callable, Dict, List, NewType, Optional, Tuple, Union
-
+import random
 import torch
 from torch.nn.utils.rnn import pad_sequence
 
@@ -124,25 +124,90 @@ class DataCollatorForLanguageModeling:
 
     tokenizer: PreTrainedTokenizerBase
     mlm: bool = True
+    wwm: bool = True
     mlm_probability: float = 0.15
 
     def __call__(
-        self, examples: List[Union[List[int], torch.Tensor, Dict[str, torch.Tensor]]]
+            self, examples: List[Union[List[int], torch.Tensor, Dict[str, torch.Tensor]]]
     ) -> Dict[str, torch.Tensor]:
         if isinstance(examples[0], (dict, BatchEncoding)):
-            examples = [e["input_ids"] for e in examples]
-        batch = self._tensorize_batch(examples)
+            input_ids = [e["input_ids"] for e in examples]
+        else:
+            input_ids = examples
+            examples = [{"input_ids":e} for e in examples]
+
+        batch_input = self._tensorize_batch(input_ids)
+
         if self.mlm:
-            inputs, labels = self.mask_tokens(batch)
+            if self.wwm:  # Whole Word Mask
+                mask_labels = []
+                for e in examples:
+                    ref_tokens = []
+                    for id in e['input_ids'].tolist():
+                        token = self.tokenizer._convert_id_to_token(id)
+                        ref_tokens.append(token)
+
+                    # For Chinese tokens, we need extra inf to mark sub-word, e.g [喜,欢]-> [喜，##欢]
+                    if "chinese_ref" in e:
+                        ref_pos = e['chinese_ref'].tolist()
+                        for i in range(e['input_ids'].size(0)):
+                            if i in ref_pos:
+                                ref_tokens[i] = '##' + ref_tokens[i]
+                    mask_labels.append(self._whole_word_mask(ref_tokens))
+                batch_mask = self._tensorize_batch(mask_labels)
+                inputs, labels = self.mask_tokens(batch_input, batch_mask)
+            else:
+                inputs, labels = self.mask_tokens(batch_input)
             return {"input_ids": inputs, "labels": labels}
         else:
-            labels = batch.clone().detach()
+            labels = batch_input.clone().detach()
             if self.tokenizer.pad_token_id is not None:
                 labels[labels == self.tokenizer.pad_token_id] = -100
-            return {"input_ids": batch, "labels": labels}
+            return {"input_ids": batch_input, "labels": labels}
+
+    def _whole_word_mask(self, input_tokens: List[str], max_predictions=512):
+        """
+        Get 0/1 labels for masked tokens with whole word mask proxy
+        """
+
+        cand_indexes = []
+        for (i, token) in enumerate(input_tokens):
+            if token == "[CLS]" or token == "[SEP]":
+                continue
+
+            if len(cand_indexes) >= 1 and token.startswith("##"):
+                cand_indexes[-1].append(i)
+            else:
+                cand_indexes.append([i])
+
+        random.shuffle(cand_indexes)
+        num_to_predict = min(max_predictions, max(1, int(round(len(input_tokens) * self.mlm_probability))))
+        masked_lms = []
+        covered_indexes = set()
+        for index_set in cand_indexes:
+            if len(masked_lms) >= num_to_predict:
+                break
+            # If adding a whole-word mask would exceed the maximum number of
+            # predictions, then just skip this candidate.
+            if len(masked_lms) + len(index_set) > num_to_predict:
+                continue
+            is_any_index_covered = False
+            for index in index_set:
+                if index in covered_indexes:
+                    is_any_index_covered = True
+                    break
+            if is_any_index_covered:
+                continue
+            for index in index_set:
+                covered_indexes.add(index)
+                masked_lms.append(index)
+
+        assert len(covered_indexes) == len(masked_lms)
+        mask_labels = [1 if i in covered_indexes else 0 for i in range(len(input_tokens))]
+        return mask_labels
 
     def _tensorize_batch(
-        self, examples: List[Union[List[int], torch.Tensor, Dict[str, torch.Tensor]]]
+            self, examples: List[Union[List[int], torch.Tensor, Dict[str, torch.Tensor]]]
     ) -> torch.Tensor:
         # In order to accept both lists of lists and lists of Tensors
         if isinstance(examples[0], (list, tuple)):
@@ -159,19 +224,22 @@ def _tensorize_batch(
                 )
             return pad_sequence(examples, batch_first=True, padding_value=self.tokenizer.pad_token_id)
 
-    def mask_tokens(self, inputs: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
+    def mask_tokens(self, inputs: torch.Tensor, mask_labels: torch.Tensor = None) -> Tuple[torch.Tensor, torch.Tensor]:
         """
         Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original.
+        Set 'mask_labels' means we use whole word mask (wwm), we directly mask idxs according to it's ref.
         """
 
         if self.tokenizer.mask_token is None:
             raise ValueError(
                 "This tokenizer does not have a mask token which is necessary for masked language modeling. Remove the --mlm flag if you want to use this tokenizer."
             )
-
         labels = inputs.clone()
         # We sample a few tokens in each sequence for masked-LM training (with probability args.mlm_probability defaults to 0.15 in Bert/RoBERTa)
-        probability_matrix = torch.full(labels.shape, self.mlm_probability)
+        if self.wwm and mask_labels is not None:
+            probability_matrix = mask_labels
+        else:
+            probability_matrix = torch.full(labels.shape, self.mlm_probability)
         special_tokens_mask = [
             self.tokenizer.get_special_tokens_mask(val, already_has_special_tokens=True) for val in labels.tolist()
         ]
@@ -179,7 +247,10 @@ def mask_tokens(self, inputs: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]
         if self.tokenizer._pad_token is not None:
             padding_mask = labels.eq(self.tokenizer.pad_token_id)
             probability_matrix.masked_fill_(padding_mask, value=0.0)
-        masked_indices = torch.bernoulli(probability_matrix).bool()
+        if self.wwm:
+            masked_indices = probability_matrix.bool()
+        else:
+            masked_indices = torch.bernoulli(probability_matrix).bool()
         labels[~masked_indices] = -100  # We only compute loss on masked tokens
 
         # 80% of the time, we replace masked input tokens with tokenizer.mask_token ([MASK])
diff --git a/src/transformers/data/datasets/language_modeling.py b/src/transformers/data/datasets/language_modeling.py
index 17f4ae0a5068..7b70c8af6e6a 100644
--- a/src/transformers/data/datasets/language_modeling.py
+++ b/src/transformers/data/datasets/language_modeling.py
@@ -3,7 +3,7 @@
 import random
 import time
 from typing import Dict, List, Optional
-
+import json
 import torch
 from torch.utils.data.dataset import Dataset
 
@@ -106,12 +106,47 @@ def __init__(self, tokenizer: PreTrainedTokenizer, file_path: str, block_size: i
 
         batch_encoding = tokenizer(lines, add_special_tokens=True, truncation=True, max_length=block_size)
         self.examples = batch_encoding["input_ids"]
+        self.examples = [{"input_ids": torch.tensor(e, dtype=torch.long)} for e in self.examples]
 
     def __len__(self):
         return len(self.examples)
 
-    def __getitem__(self, i) -> torch.Tensor:
-        return torch.tensor(self.examples[i], dtype=torch.long)
+    def __getitem__(self, i) -> Dict[str, torch.tensor]:
+        return self.examples[i]
+
+class LineByLineWithRefDataset(Dataset):
+    """
+    This will be superseded by a framework-agnostic approach
+    soon.
+    """
+
+    def __init__(self, tokenizer: PreTrainedTokenizer, file_path: str, block_size: int, ref_path: str):
+        assert os.path.isfile(file_path), f"Input file path {file_path} not found"
+        assert os.path.isfile(ref_path), f"Ref file path {file_path} not found"
+        # Here, we do not cache the features, operating under the assumption
+        # that we will soon use fast multithreaded tokenizers from the
+        # `tokenizers` repo everywhere =)
+        logger.info("Creating features from dataset file at %s", file_path)
+        logger.info("Use ref segment results at %s", ref_path)
+        with open(file_path, encoding="utf-8") as f:
+            data = [line for line in f.read().splitlines() if (len(line) > 0 and not line.isspace())]
+        batch_encoding = tokenizer(data, add_special_tokens=True, truncation=True, max_length=block_size)
+        self.examples = batch_encoding["input_ids"]
+        self.examples = [{"input_ids": torch.tensor(e, dtype=torch.long)} for e in self.examples]
+
+        # Get ref inf from file
+        with open(ref_path, encoding="utf-8") as f:
+            ref = [json.loads(line) for line in f.read().splitlines() if (len(line) > 0 and not line.isspace())]
+        assert len(data) == len(ref)
+        n = len(self.examples)
+        for i in range(n):
+            self.examples[i]['chinese_ref'] = torch.tensor(ref[i], dtype=torch.long)
+
+    def __len__(self):
+        return len(self.examples)
+
+    def __getitem__(self, i) -> Dict[str, torch.tensor]:
+        return self.examples[i]
 
 
 class LineByLineWithSOPTextDataset(Dataset):

From eeb0e0232c22026f33487433f9a6fde20d561c4a Mon Sep 17 00:00:00 2001
From: Hgtc <hgtcwl@foxmail.com>
Date: Tue, 20 Oct 2020 18:01:36 +0800
Subject: [PATCH 02/31] MOD: adjust format

---
 examples/language-modeling/run_language_modeling.py | 2 +-
 src/transformers/data/data_collator.py              | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/examples/language-modeling/run_language_modeling.py b/examples/language-modeling/run_language_modeling.py
index f8ec07c9cad1..94efa4c06791 100644
--- a/examples/language-modeling/run_language_modeling.py
+++ b/examples/language-modeling/run_language_modeling.py
@@ -103,7 +103,7 @@ class DataTrainingArguments:
         metadata={"help": "An optional input evaluation data file to evaluate the perplexity on (a text file)."},
     )
     chinese_ref_file: Optional[str] = field(
-        default=False,
+        default=NoneŒ,
         metadata={"help": "An optional input ref data file for whole word mask(wwm) in Chinees."},
     )
     line_by_line: bool = field(
diff --git a/src/transformers/data/data_collator.py b/src/transformers/data/data_collator.py
index d8b6f687333f..3018f6d2ec32 100644
--- a/src/transformers/data/data_collator.py
+++ b/src/transformers/data/data_collator.py
@@ -128,7 +128,7 @@ class DataCollatorForLanguageModeling:
     mlm_probability: float = 0.15
 
     def __call__(
-            self, examples: List[Union[List[int], torch.Tensor, Dict[str, torch.Tensor]]]
+        self, examples: List[Union[List[int], torch.Tensor, Dict[str, torch.Tensor]]]
     ) -> Dict[str, torch.Tensor]:
         if isinstance(examples[0], (dict, BatchEncoding)):
             input_ids = [e["input_ids"] for e in examples]
@@ -207,7 +207,7 @@ def _whole_word_mask(self, input_tokens: List[str], max_predictions=512):
         return mask_labels
 
     def _tensorize_batch(
-            self, examples: List[Union[List[int], torch.Tensor, Dict[str, torch.Tensor]]]
+        self, examples: List[Union[List[int], torch.Tensor, Dict[str, torch.Tensor]]]
     ) -> torch.Tensor:
         # In order to accept both lists of lists and lists of Tensors
         if isinstance(examples[0], (list, tuple)):

From dde73e2a1046ee216d4705b2450d8ada2754bfc8 Mon Sep 17 00:00:00 2001
From: Hgtc <hgtcwl@foxmail.com>
Date: Tue, 20 Oct 2020 18:31:26 +0800
Subject: [PATCH 03/31] MOD: reformat code

---
 examples/language-modeling/chinese_ref.py     | 55 ++++++++++---------
 .../run_language_modeling.py                  | 19 +++----
 src/transformers/data/data_collator.py        | 10 ++--
 .../data/datasets/language_modeling.py        | 16 ++----
 4 files changed, 48 insertions(+), 52 deletions(-)

diff --git a/examples/language-modeling/chinese_ref.py b/examples/language-modeling/chinese_ref.py
index e0bfbd1add05..b84ab02b42fd 100644
--- a/examples/language-modeling/chinese_ref.py
+++ b/examples/language-modeling/chinese_ref.py
@@ -18,14 +18,14 @@ def _is_chinese_char(cp):
     # space-separated words, so they are not treated specially and handled
     # like the all of the other languages.
     if (
-            (cp >= 0x4E00 and cp <= 0x9FFF)
-            or (cp >= 0x3400 and cp <= 0x4DBF)  #
-            or (cp >= 0x20000 and cp <= 0x2A6DF)  #
-            or (cp >= 0x2A700 and cp <= 0x2B73F)  #
-            or (cp >= 0x2B740 and cp <= 0x2B81F)  #
-            or (cp >= 0x2B820 and cp <= 0x2CEAF)  #
-            or (cp >= 0xF900 and cp <= 0xFAFF)
-            or (cp >= 0x2F800 and cp <= 0x2FA1F)  #
+        (cp >= 0x4E00 and cp <= 0x9FFF)
+        or (cp >= 0x3400 and cp <= 0x4DBF)  #
+        or (cp >= 0x20000 and cp <= 0x2A6DF)  #
+        or (cp >= 0x2A700 and cp <= 0x2B73F)  #
+        or (cp >= 0x2B740 and cp <= 0x2B81F)  #
+        or (cp >= 0x2B820 and cp <= 0x2CEAF)  #
+        or (cp >= 0xF900 and cp <= 0xFAFF)
+        or (cp >= 0x2F800 and cp <= 0x2FA1F)  #
     ):  #
         return True
 
@@ -64,10 +64,10 @@ def add_sub_symbol(bert_tokens: List[str], chinese_word_set: set()):
         if is_chinese(bert_word[start]):
             l = min(end - start, max_word_len)
             for i in range(l, 1, -1):
-                whole_word = ''.join(bert_word[start:start + i])
+                whole_word = "".join(bert_word[start : start + i])
                 if whole_word in chinese_word_set:
                     for j in range(start + 1, start + i):
-                        bert_word[j] = '##' + bert_word[j]
+                        bert_word[j] = "##" + bert_word[j]
                     start = start + i
                     single_word = False
                     break
@@ -80,15 +80,15 @@ def prepare_ref(lines: List[str], ltp_tokenizer: LTP, bert_tokenizer: BertTokeni
     ltp_res = []
 
     for i in range(0, len(lines), 100):
-        res = ltp_tokenizer.seg(lines[i:i + 100])[0]
+        res = ltp_tokenizer.seg(lines[i : i + 100])[0]
         res = [get_chinese_word(r) for r in res]
         ltp_res.extend(res)
     assert len(ltp_res) == len(lines)
 
     bert_res = []
     for i in range(0, len(lines), 100):
-        res = bert_tokenizer(lines[i:i + 100], add_special_tokens=True, truncation=True, max_length=512)
-        bert_res.extend(res['input_ids'])
+        res = bert_tokenizer(lines[i : i + 100], add_special_tokens=True, truncation=True, max_length=512)
+        bert_res.extend(res["input_ids"])
     assert len(bert_res) == len(lines)
 
     ref_ids = []
@@ -102,7 +102,7 @@ def prepare_ref(lines: List[str], ltp_tokenizer: LTP, bert_tokenizer: BertTokeni
         ref_id = []
         # We only save pos of chinese subwords start with ##, which mean is part of a whole word.
         for i, token in enumerate(input_tokens):
-            if token[:2] == '##':
+            if token[:2] == "##":
                 clean_token = token[2:]
                 # save chinese tokens' pos
                 if len(clean_token) == 1 and _is_chinese_char(ord(clean_token)):
@@ -117,7 +117,7 @@ def prepare_ref(lines: List[str], ltp_tokenizer: LTP, bert_tokenizer: BertTokeni
 def main(args):
     # For Chinese (Ro)Bert, the best result is from : RoBERTa-wwm-ext (https://github.com/ymcui/Chinese-BERT-wwm)
     # If we want to fine-tune these model, we have to use same tokenizer : LTP (https://github.com/HIT-SCIR/ltp)
-    with open(args.file_name, 'r', encoding='utf-8') as f:
+    with open(args.file_name, "r", encoding="utf-8") as f:
         data = f.readlines()
 
     ltp_tokenizer = LTP(args.ltp)  # faster in GPU device
@@ -125,19 +125,24 @@ def main(args):
 
     ref_ids = prepare_ref(data, ltp_tokenizer, bert_tokenizer)
 
-    with open(args.save_path, 'w', encoding='utf-8') as f:
-        data = [json.dumps(ref) + '\n' for ref in ref_ids]
+    with open(args.save_path, "w", encoding="utf-8") as f:
+        data = [json.dumps(ref) + "\n" for ref in ref_ids]
         f.writelines(data)
 
 
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description='prepare_chinese_ref')
-    parser.add_argument('--file_name', type=str, default='./resources/chinese-demo.txt',
-                        help='file need process, same as training data in lm')
-    parser.add_argument('--ltp', type=str, default='./resources/ltp',
-                        help='resources for LTP tokenizer, usually a path')
-    parser.add_argument('--bert', type=str, default='./resources/robert', help='resources for Bert tokenizer')
-    parser.add_argument('--save_path', type=str, default='./resources/ref.txt', help='path to save res')
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="prepare_chinese_ref")
+    parser.add_argument(
+        "--file_name",
+        type=str,
+        default="./resources/chinese-demo.txt",
+        help="file need process, same as training data in lm",
+    )
+    parser.add_argument(
+        "--ltp", type=str, default="./resources/ltp", help="resources for LTP tokenizer, usually a path"
+    )
+    parser.add_argument("--bert", type=str, default="./resources/robert", help="resources for Bert tokenizer")
+    parser.add_argument("--save_path", type=str, default="./resources/ref.txt", help="path to save res")
 
     args = parser.parse_args()
     main(args)
diff --git a/examples/language-modeling/run_language_modeling.py b/examples/language-modeling/run_language_modeling.py
index 94efa4c06791..01ca3eb4dc7e 100644
--- a/examples/language-modeling/run_language_modeling.py
+++ b/examples/language-modeling/run_language_modeling.py
@@ -103,8 +103,7 @@ class DataTrainingArguments:
         metadata={"help": "An optional input evaluation data file to evaluate the perplexity on (a text file)."},
     )
     chinese_ref_file: Optional[str] = field(
-        default=NoneŒ,
-        metadata={"help": "An optional input ref data file for whole word mask(wwm) in Chinees."},
+        default=NoneŒ, metadata={"help": "An optional input ref data file for whole word mask(wwm) in Chinees."},
     )
     line_by_line: bool = field(
         default=False,
@@ -114,9 +113,7 @@ class DataTrainingArguments:
     mlm: bool = field(
         default=False, metadata={"help": "Train with masked-language modeling loss instead of language modeling."}
     )
-    wwm: bool = field(
-        default=False, metadata={"help": "Use Whole Word Mask."}
-    )
+    wwm: bool = field(default=False, metadata={"help": "Use Whole Word Mask."})
     mlm_probability: float = field(
         default=0.15, metadata={"help": "Ratio of tokens to mask for masked language modeling loss"}
     )
@@ -152,8 +149,12 @@ def get_dataset(
     def _dataset(file_path):
         if args.line_by_line:
             if args.chinese_ref_file:
-                return LineByLineWithRefDataset(tokenizer=tokenizer, file_path=file_path, block_size=args.block_size,
-                                             ref_path=args.chinese_ref_file)
+                return LineByLineWithRefDataset(
+                    tokenizer=tokenizer,
+                    file_path=file_path,
+                    block_size=args.block_size,
+                    ref_path=args.chinese_ref_file,
+                )
 
             return LineByLineTextDataset(tokenizer=tokenizer, file_path=file_path, block_size=args.block_size)
         else:
@@ -277,9 +278,7 @@ def main():
     )
     if config.model_type == "xlnet":
         data_collator = DataCollatorForPermutationLanguageModeling(
-            tokenizer=tokenizer,
-            plm_probability=data_args.plm_probability,
-            max_span_length=data_args.max_span_length,
+            tokenizer=tokenizer, plm_probability=data_args.plm_probability, max_span_length=data_args.max_span_length,
         )
     else:
         data_collator = DataCollatorForLanguageModeling(
diff --git a/src/transformers/data/data_collator.py b/src/transformers/data/data_collator.py
index 3018f6d2ec32..ea71311fc446 100644
--- a/src/transformers/data/data_collator.py
+++ b/src/transformers/data/data_collator.py
@@ -134,7 +134,7 @@ def __call__(
             input_ids = [e["input_ids"] for e in examples]
         else:
             input_ids = examples
-            examples = [{"input_ids":e} for e in examples]
+            examples = [{"input_ids": e} for e in examples]
 
         batch_input = self._tensorize_batch(input_ids)
 
@@ -143,16 +143,16 @@ def __call__(
                 mask_labels = []
                 for e in examples:
                     ref_tokens = []
-                    for id in e['input_ids'].tolist():
+                    for id in e["input_ids"].tolist():
                         token = self.tokenizer._convert_id_to_token(id)
                         ref_tokens.append(token)
 
                     # For Chinese tokens, we need extra inf to mark sub-word, e.g [喜,欢]-> [喜，##欢]
                     if "chinese_ref" in e:
-                        ref_pos = e['chinese_ref'].tolist()
-                        for i in range(e['input_ids'].size(0)):
+                        ref_pos = e["chinese_ref"].tolist()
+                        for i in range(e["input_ids"].size(0)):
                             if i in ref_pos:
-                                ref_tokens[i] = '##' + ref_tokens[i]
+                                ref_tokens[i] = "##" + ref_tokens[i]
                     mask_labels.append(self._whole_word_mask(ref_tokens))
                 batch_mask = self._tensorize_batch(mask_labels)
                 inputs, labels = self.mask_tokens(batch_input, batch_mask)
diff --git a/src/transformers/data/datasets/language_modeling.py b/src/transformers/data/datasets/language_modeling.py
index 7b70c8af6e6a..5b81466cb983 100644
--- a/src/transformers/data/datasets/language_modeling.py
+++ b/src/transformers/data/datasets/language_modeling.py
@@ -37,11 +37,7 @@ def __init__(
         directory, filename = os.path.split(file_path)
         cached_features_file = os.path.join(
             cache_dir if cache_dir is not None else directory,
-            "cached_lm_{}_{}_{}".format(
-                tokenizer.__class__.__name__,
-                str(block_size),
-                filename,
-            ),
+            "cached_lm_{}_{}_{}".format(tokenizer.__class__.__name__, str(block_size), filename,),
         )
 
         # Make sure only the first process in distributed training processes the dataset,
@@ -114,6 +110,7 @@ def __len__(self):
     def __getitem__(self, i) -> Dict[str, torch.tensor]:
         return self.examples[i]
 
+
 class LineByLineWithRefDataset(Dataset):
     """
     This will be superseded by a framework-agnostic approach
@@ -140,7 +137,7 @@ def __init__(self, tokenizer: PreTrainedTokenizer, file_path: str, block_size: i
         assert len(data) == len(ref)
         n = len(self.examples)
         for i in range(n):
-            self.examples[i]['chinese_ref'] = torch.tensor(ref[i], dtype=torch.long)
+            self.examples[i]["chinese_ref"] = torch.tensor(ref[i], dtype=torch.long)
 
     def __len__(self):
         return len(self.examples)
@@ -313,12 +310,7 @@ def __init__(
 
         directory, filename = os.path.split(file_path)
         cached_features_file = os.path.join(
-            directory,
-            "cached_nsp_{}_{}_{}".format(
-                tokenizer.__class__.__name__,
-                str(block_size),
-                filename,
-            ),
+            directory, "cached_nsp_{}_{}_{}".format(tokenizer.__class__.__name__, str(block_size), filename,),
         )
 
         self.tokenizer = tokenizer

From 6470d5b6eddba09a983f58d6c0c1bea8af47a117 Mon Sep 17 00:00:00 2001
From: Hgtc <hgtcwl@foxmail.com>
Date: Tue, 20 Oct 2020 18:45:08 +0800
Subject: [PATCH 04/31] MOD: update import

---
 src/transformers/data/datasets/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/transformers/data/datasets/__init__.py b/src/transformers/data/datasets/__init__.py
index c482be98772f..7a5cb1b30399 100644
--- a/src/transformers/data/datasets/__init__.py
+++ b/src/transformers/data/datasets/__init__.py
@@ -5,6 +5,7 @@
 from .glue import GlueDataset, GlueDataTrainingArguments
 from .language_modeling import (
     LineByLineTextDataset,
+    LineByLineWithRefDataset,Œ
     LineByLineWithSOPTextDataset,
     TextDataset,
     TextDatasetForNextSentencePrediction,

From 30928b0c94ab28cd30ac93179b072df8ec91f568 Mon Sep 17 00:00:00 2001
From: Hgtc <hgtcwl@foxmail.com>
Date: Tue, 20 Oct 2020 18:49:43 +0800
Subject: [PATCH 05/31] MOD: fix bug

---
 src/transformers/data/datasets/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/transformers/data/datasets/__init__.py b/src/transformers/data/datasets/__init__.py
index 7a5cb1b30399..0cb518a715cf 100644
--- a/src/transformers/data/datasets/__init__.py
+++ b/src/transformers/data/datasets/__init__.py
@@ -5,7 +5,7 @@
 from .glue import GlueDataset, GlueDataTrainingArguments
 from .language_modeling import (
     LineByLineTextDataset,
-    LineByLineWithRefDataset,Œ
+    LineByLineWithRefDataset,
     LineByLineWithSOPTextDataset,
     TextDataset,
     TextDatasetForNextSentencePrediction,

From dc7794fd30bd4e57ea538542d959703c16195b39 Mon Sep 17 00:00:00 2001
From: Hgtc <hgtcwl@foxmail.com>
Date: Tue, 20 Oct 2020 19:11:44 +0800
Subject: [PATCH 06/31] MOD: add import

---
 src/transformers/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py
index 599ec72773a2..9398dcee7e95 100755
--- a/src/transformers/__init__.py
+++ b/src/transformers/__init__.py
@@ -291,6 +291,7 @@
         GlueDataset,
         GlueDataTrainingArguments,
         LineByLineTextDataset,
+        LineByLineWithRefDataset,
         LineByLineWithSOPTextDataset,
         SquadDataset,
         SquadDataTrainingArguments,

From 1fd0e153f056e5e7f6d985cf4b468a38d03cf860 Mon Sep 17 00:00:00 2001
From: Hgtc <hgtcwl@foxmail.com>
Date: Tue, 20 Oct 2020 19:21:08 +0800
Subject: [PATCH 07/31] MOD: fix bug

---
 examples/language-modeling/run_language_modeling.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/language-modeling/run_language_modeling.py b/examples/language-modeling/run_language_modeling.py
index 01ca3eb4dc7e..b8667376db37 100644
--- a/examples/language-modeling/run_language_modeling.py
+++ b/examples/language-modeling/run_language_modeling.py
@@ -103,7 +103,7 @@ class DataTrainingArguments:
         metadata={"help": "An optional input evaluation data file to evaluate the perplexity on (a text file)."},
     )
     chinese_ref_file: Optional[str] = field(
-        default=NoneŒ, metadata={"help": "An optional input ref data file for whole word mask(wwm) in Chinees."},
+        default=None, metadata={"help": "An optional input ref data file for whole word mask(wwm) in Chinees."},
     )
     line_by_line: bool = field(
         default=False,

From 0aec80a8fd24f220fb6cf9ad8bd97ce050aff86f Mon Sep 17 00:00:00 2001
From: Hgtc <hgtcwl@foxmail.com>
Date: Tue, 20 Oct 2020 22:19:29 +0800
Subject: [PATCH 08/31] MOD: decouple code and update readme

---
 examples/language-modeling/README.md          |  40 ++++-
 .../run_language_modeling.py                  |  21 ++-
 src/transformers/__init__.py                  |   1 +
 src/transformers/data/data_collator.py        | 155 +++++++++++-------
 4 files changed, 156 insertions(+), 61 deletions(-)

diff --git a/examples/language-modeling/README.md b/examples/language-modeling/README.md
index a66215351a7d..6ebcfd802e33 100644
--- a/examples/language-modeling/README.md
+++ b/examples/language-modeling/README.md
@@ -45,9 +45,45 @@ slightly slower (over-fitting takes more epochs).
 
 We use the `--mlm` flag so that the script may change its loss function.
 
+If use whole-word masking, use both `--mlm` and `--wwm` flag(for English Model).
+
+```bash
+export TRAIN_FILE=/path/to/dataset/wiki.train.raw
+export TEST_FILE=/path/to/dataset/wiki.test.raw
+
+python run_language_modeling.py \
+    --output_dir=output \
+    --model_type=roberta \
+    --model_name_or_path=roberta-base \
+    --do_train \
+    --train_data_file=$TRAIN_FILE \
+    --do_eval \
+    --eval_data_file=$TEST_FILE \
+    --mlm \
+    --wwm
+```
+
+For Chinese Model, we need to generate ref files, case it's char level.
+
+```bash
+export TRAIN_FILE=/path/to/dataset/wiki.train.raw
+export LTP_RESOURCE=/path/to/ltp/tokenizer
+export BERT_RESOURCE=/path/to/bert/tokenizer
+export SAVE_PATH=/path/to/data/ref.txt
+
+python chinese_ref.py \
+    --file_name=$TRAIN_FILE \
+    --ltp=$LTP_RESOURCE
+    --bert=$BERT_RESOURCE \
+    --save_path=$SAVE_PATH 
+```
+Then: 
+
+
 ```bash
 export TRAIN_FILE=/path/to/dataset/wiki.train.raw
 export TEST_FILE=/path/to/dataset/wiki.test.raw
+export REF_FILE=/path/to/ref.txt
 
 python run_language_modeling.py \
     --output_dir=output \
@@ -55,9 +91,11 @@ python run_language_modeling.py \
     --model_name_or_path=roberta-base \
     --do_train \
     --train_data_file=$TRAIN_FILE \
+    --chinese_ref_file=$REF_FILE \
     --do_eval \
     --eval_data_file=$TEST_FILE \
-    --mlm
+    --mlm \
+    --wwm
 ```
 
 ### XLNet and permutation language modeling
diff --git a/examples/language-modeling/run_language_modeling.py b/examples/language-modeling/run_language_modeling.py
index b8667376db37..2f5995a873e9 100644
--- a/examples/language-modeling/run_language_modeling.py
+++ b/examples/language-modeling/run_language_modeling.py
@@ -36,6 +36,7 @@
     AutoModelWithLMHead,
     AutoTokenizer,
     DataCollatorForLanguageModeling,
+    DataCollatorForWholeWordMask,
     DataCollatorForPermutationLanguageModeling,
     HfArgumentParser,
     LineByLineTextDataset,
@@ -113,7 +114,9 @@ class DataTrainingArguments:
     mlm: bool = field(
         default=False, metadata={"help": "Train with masked-language modeling loss instead of language modeling."}
     )
-    wwm: bool = field(default=False, metadata={"help": "Use Whole Word Mask."})
+    wwm: bool = field(
+        default=False, metadata={"help": "Use Whole Word Mask."}
+    )
     mlm_probability: float = field(
         default=0.15, metadata={"help": "Ratio of tokens to mask for masked language modeling loss"}
     )
@@ -149,6 +152,10 @@ def get_dataset(
     def _dataset(file_path):
         if args.line_by_line:
             if args.chinese_ref_file:
+                if not args.wwm or args.mlm:
+                    raise ValueError(
+                        "Need set wwm and mlm to true for Chinese Whole Word Mask"
+                    )
                 return LineByLineWithRefDataset(
                     tokenizer=tokenizer,
                     file_path=file_path,
@@ -187,7 +194,6 @@ def main():
             "Cannot do evaluation without an evaluation data file. Either supply a file to --eval_data_file "
             "or remove the --do_eval argument."
         )
-
     if (
         os.path.exists(training_args.output_dir)
         and os.listdir(training_args.output_dir)
@@ -281,9 +287,14 @@ def main():
             tokenizer=tokenizer, plm_probability=data_args.plm_probability, max_span_length=data_args.max_span_length,
         )
     else:
-        data_collator = DataCollatorForLanguageModeling(
-            tokenizer=tokenizer, mlm=data_args.mlm, wwm=data_args.wwm, mlm_probability=data_args.mlm_probability
-        )
+        if data_args.mlm and data_args.wwm:
+            data_collator = DataCollatorForWholeWordMask(
+                tokenizer=tokenizer, mlm_probability=data_args.mlm_probability
+            )
+        else:
+            data_collator = DataCollatorForLanguageModeling(
+                tokenizer=tokenizer, mlm=data_args.mlm, mlm_probability=data_args.mlm_probability
+            )
 
     # Initialize our Trainer
     trainer = Trainer(
diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py
index 9398dcee7e95..8537cb431185 100755
--- a/src/transformers/__init__.py
+++ b/src/transformers/__init__.py
@@ -281,6 +281,7 @@
     from .data.data_collator import (
         DataCollator,
         DataCollatorForLanguageModeling,
+        DataCollatorForWholeWordMask,
         DataCollatorForNextSentencePrediction,
         DataCollatorForPermutationLanguageModeling,
         DataCollatorForSOP,
diff --git a/src/transformers/data/data_collator.py b/src/transformers/data/data_collator.py
index ea71311fc446..236db16ccd49 100644
--- a/src/transformers/data/data_collator.py
+++ b/src/transformers/data/data_collator.py
@@ -113,7 +113,6 @@ def __call__(self, features: List[Dict[str, Union[List[int], torch.Tensor]]]) ->
             del batch["label_ids"]
         return batch
 
-
 @dataclass
 class DataCollatorForLanguageModeling:
     """
@@ -124,11 +123,86 @@ class DataCollatorForLanguageModeling:
 
     tokenizer: PreTrainedTokenizerBase
     mlm: bool = True
-    wwm: bool = True
     mlm_probability: float = 0.15
 
     def __call__(
         self, examples: List[Union[List[int], torch.Tensor, Dict[str, torch.Tensor]]]
+    ) -> Dict[str, torch.Tensor]:
+        if isinstance(examples[0], (dict, BatchEncoding)):
+            examples = [e["input_ids"] for e in examples]
+        batch = self._tensorize_batch(examples)
+        if self.mlm:
+            inputs, labels = self.mask_tokens(batch)
+            return {"input_ids": inputs, "labels": labels}
+        else:
+            labels = batch.clone().detach()
+            if self.tokenizer.pad_token_id is not None:
+                labels[labels == self.tokenizer.pad_token_id] = -100
+            return {"input_ids": batch, "labels": labels}
+
+    def _tensorize_batch(
+        self, examples: List[Union[List[int], torch.Tensor, Dict[str, torch.Tensor]]]
+    ) -> torch.Tensor:
+        # In order to accept both lists of lists and lists of Tensors
+        if isinstance(examples[0], (list, tuple)):
+            examples = [torch.tensor(e, dtype=torch.long) for e in examples]
+        length_of_first = examples[0].size(0)
+        are_tensors_same_length = all(x.size(0) == length_of_first for x in examples)
+        if are_tensors_same_length:
+            return torch.stack(examples, dim=0)
+        else:
+            if self.tokenizer._pad_token is None:
+                raise ValueError(
+                    "You are attempting to pad samples but the tokenizer you are using"
+                    f" ({self.tokenizer.__class__.__name__}) does not have one."
+                )
+            return pad_sequence(examples, batch_first=True, padding_value=self.tokenizer.pad_token_id)
+
+    def mask_tokens(self, inputs: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
+        """
+        Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original.
+        """
+
+        if self.tokenizer.mask_token is None:
+            raise ValueError(
+                "This tokenizer does not have a mask token which is necessary for masked language modeling. Remove the --mlm flag if you want to use this tokenizer."
+            )
+
+        labels = inputs.clone()
+        # We sample a few tokens in each sequence for masked-LM training (with probability args.mlm_probability defaults to 0.15 in Bert/RoBERTa)
+        probability_matrix = torch.full(labels.shape, self.mlm_probability)
+        special_tokens_mask = [
+            self.tokenizer.get_special_tokens_mask(val, already_has_special_tokens=True) for val in labels.tolist()
+        ]
+        probability_matrix.masked_fill_(torch.tensor(special_tokens_mask, dtype=torch.bool), value=0.0)
+        if self.tokenizer._pad_token is not None:
+            padding_mask = labels.eq(self.tokenizer.pad_token_id)
+            probability_matrix.masked_fill_(padding_mask, value=0.0)
+        masked_indices = torch.bernoulli(probability_matrix).bool()
+        labels[~masked_indices] = -100  # We only compute loss on masked tokens
+
+        # 80% of the time, we replace masked input tokens with tokenizer.mask_token ([MASK])
+        indices_replaced = torch.bernoulli(torch.full(labels.shape, 0.8)).bool() & masked_indices
+        inputs[indices_replaced] = self.tokenizer.convert_tokens_to_ids(self.tokenizer.mask_token)
+
+        # 10% of the time, we replace masked input tokens with random word
+        indices_random = torch.bernoulli(torch.full(labels.shape, 0.5)).bool() & masked_indices & ~indices_replaced
+        random_words = torch.randint(len(self.tokenizer), labels.shape, dtype=torch.long)
+        inputs[indices_random] = random_words[indices_random]
+
+        # The rest of the time (10% of the time) we keep the masked input tokens unchanged
+        return inputs, labels
+
+@dataclass
+class DataCollatorForWholeWordMask(DataCollatorForLanguageModeling):
+    """
+    Data collator used for language modeling.
+    - collates batches of tensors, honoring their tokenizer's pad_token
+    - preprocesses batches for masked language modeling
+    """
+
+    def __call__(
+            self, examples: List[Union[List[int], torch.Tensor, Dict[str, torch.Tensor]]]
     ) -> Dict[str, torch.Tensor]:
         if isinstance(examples[0], (dict, BatchEncoding)):
             input_ids = [e["input_ids"] for e in examples]
@@ -138,32 +212,24 @@ def __call__(
 
         batch_input = self._tensorize_batch(input_ids)
 
-        if self.mlm:
-            if self.wwm:  # Whole Word Mask
-                mask_labels = []
-                for e in examples:
-                    ref_tokens = []
-                    for id in e["input_ids"].tolist():
-                        token = self.tokenizer._convert_id_to_token(id)
-                        ref_tokens.append(token)
-
-                    # For Chinese tokens, we need extra inf to mark sub-word, e.g [喜,欢]-> [喜，##欢]
-                    if "chinese_ref" in e:
-                        ref_pos = e["chinese_ref"].tolist()
-                        for i in range(e["input_ids"].size(0)):
-                            if i in ref_pos:
-                                ref_tokens[i] = "##" + ref_tokens[i]
-                    mask_labels.append(self._whole_word_mask(ref_tokens))
-                batch_mask = self._tensorize_batch(mask_labels)
-                inputs, labels = self.mask_tokens(batch_input, batch_mask)
-            else:
-                inputs, labels = self.mask_tokens(batch_input)
-            return {"input_ids": inputs, "labels": labels}
-        else:
-            labels = batch_input.clone().detach()
-            if self.tokenizer.pad_token_id is not None:
-                labels[labels == self.tokenizer.pad_token_id] = -100
-            return {"input_ids": batch_input, "labels": labels}
+        mask_labels = []
+        for e in examples:
+            ref_tokens = []
+            for id in e["input_ids"].tolist():
+                token = self.tokenizer._convert_id_to_token(id)
+                ref_tokens.append(token)
+
+            # For Chinese tokens, we need extra inf to mark sub-word, e.g [喜,欢]-> [喜，##欢]
+            if "chinese_ref" in e:
+                ref_pos = e["chinese_ref"].tolist()
+                len_seq = e["input_ids"].size(0)
+                for i in range(len_seq):
+                    if i in ref_pos:
+                        ref_tokens[i] = "##" + ref_tokens[i]
+            mask_labels.append(self._whole_word_mask(ref_tokens))
+        batch_mask = self._tensorize_batch(mask_labels)
+        inputs, labels = self.mask_tokens(batch_input, batch_mask)
+        return {"input_ids": inputs, "labels": labels}
 
     def _whole_word_mask(self, input_tokens: List[str], max_predictions=512):
         """
@@ -206,25 +272,7 @@ def _whole_word_mask(self, input_tokens: List[str], max_predictions=512):
         mask_labels = [1 if i in covered_indexes else 0 for i in range(len(input_tokens))]
         return mask_labels
 
-    def _tensorize_batch(
-        self, examples: List[Union[List[int], torch.Tensor, Dict[str, torch.Tensor]]]
-    ) -> torch.Tensor:
-        # In order to accept both lists of lists and lists of Tensors
-        if isinstance(examples[0], (list, tuple)):
-            examples = [torch.tensor(e, dtype=torch.long) for e in examples]
-        length_of_first = examples[0].size(0)
-        are_tensors_same_length = all(x.size(0) == length_of_first for x in examples)
-        if are_tensors_same_length:
-            return torch.stack(examples, dim=0)
-        else:
-            if self.tokenizer._pad_token is None:
-                raise ValueError(
-                    "You are attempting to pad samples but the tokenizer you are using"
-                    f" ({self.tokenizer.__class__.__name__}) does not have one."
-                )
-            return pad_sequence(examples, batch_first=True, padding_value=self.tokenizer.pad_token_id)
-
-    def mask_tokens(self, inputs: torch.Tensor, mask_labels: torch.Tensor = None) -> Tuple[torch.Tensor, torch.Tensor]:
+    def mask_tokens(self, inputs: torch.Tensor, mask_labels: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
         """
         Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original.
         Set 'mask_labels' means we use whole word mask (wwm), we directly mask idxs according to it's ref.
@@ -236,10 +284,9 @@ def mask_tokens(self, inputs: torch.Tensor, mask_labels: torch.Tensor = None) ->
             )
         labels = inputs.clone()
         # We sample a few tokens in each sequence for masked-LM training (with probability args.mlm_probability defaults to 0.15 in Bert/RoBERTa)
-        if self.wwm and mask_labels is not None:
-            probability_matrix = mask_labels
-        else:
-            probability_matrix = torch.full(labels.shape, self.mlm_probability)
+
+        probability_matrix = mask_labels
+
         special_tokens_mask = [
             self.tokenizer.get_special_tokens_mask(val, already_has_special_tokens=True) for val in labels.tolist()
         ]
@@ -247,10 +294,8 @@ def mask_tokens(self, inputs: torch.Tensor, mask_labels: torch.Tensor = None) ->
         if self.tokenizer._pad_token is not None:
             padding_mask = labels.eq(self.tokenizer.pad_token_id)
             probability_matrix.masked_fill_(padding_mask, value=0.0)
-        if self.wwm:
-            masked_indices = probability_matrix.bool()
-        else:
-            masked_indices = torch.bernoulli(probability_matrix).bool()
+
+        masked_indices = probability_matrix.bool()
         labels[~masked_indices] = -100  # We only compute loss on masked tokens
 
         # 80% of the time, we replace masked input tokens with tokenizer.mask_token ([MASK])

From 156d40a1fed99024314f368fb2654597ba9a93e7 Mon Sep 17 00:00:00 2001
From: Hgtc <hgtcwl@foxmail.com>
Date: Tue, 20 Oct 2020 22:36:20 +0800
Subject: [PATCH 09/31] MOD: reformat code

---
 examples/language-modeling/run_language_modeling.py | 8 ++------
 src/transformers/data/data_collator.py              | 4 +++-
 2 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/examples/language-modeling/run_language_modeling.py b/examples/language-modeling/run_language_modeling.py
index 2f5995a873e9..77c0c3280059 100644
--- a/examples/language-modeling/run_language_modeling.py
+++ b/examples/language-modeling/run_language_modeling.py
@@ -114,9 +114,7 @@ class DataTrainingArguments:
     mlm: bool = field(
         default=False, metadata={"help": "Train with masked-language modeling loss instead of language modeling."}
     )
-    wwm: bool = field(
-        default=False, metadata={"help": "Use Whole Word Mask."}
-    )
+    wwm: bool = field(default=False, metadata={"help": "Use Whole Word Mask."})
     mlm_probability: float = field(
         default=0.15, metadata={"help": "Ratio of tokens to mask for masked language modeling loss"}
     )
@@ -153,9 +151,7 @@ def _dataset(file_path):
         if args.line_by_line:
             if args.chinese_ref_file:
                 if not args.wwm or args.mlm:
-                    raise ValueError(
-                        "Need set wwm and mlm to true for Chinese Whole Word Mask"
-                    )
+                    raise ValueError("Need set wwm and mlm to true for Chinese Whole Word Mask")
                 return LineByLineWithRefDataset(
                     tokenizer=tokenizer,
                     file_path=file_path,
diff --git a/src/transformers/data/data_collator.py b/src/transformers/data/data_collator.py
index 236db16ccd49..0b9d2b9c3701 100644
--- a/src/transformers/data/data_collator.py
+++ b/src/transformers/data/data_collator.py
@@ -113,6 +113,7 @@ def __call__(self, features: List[Dict[str, Union[List[int], torch.Tensor]]]) ->
             del batch["label_ids"]
         return batch
 
+
 @dataclass
 class DataCollatorForLanguageModeling:
     """
@@ -193,6 +194,7 @@ def mask_tokens(self, inputs: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]
         # The rest of the time (10% of the time) we keep the masked input tokens unchanged
         return inputs, labels
 
+
 @dataclass
 class DataCollatorForWholeWordMask(DataCollatorForLanguageModeling):
     """
@@ -202,7 +204,7 @@ class DataCollatorForWholeWordMask(DataCollatorForLanguageModeling):
     """
 
     def __call__(
-            self, examples: List[Union[List[int], torch.Tensor, Dict[str, torch.Tensor]]]
+        self, examples: List[Union[List[int], torch.Tensor, Dict[str, torch.Tensor]]]
     ) -> Dict[str, torch.Tensor]:
         if isinstance(examples[0], (dict, BatchEncoding)):
             input_ids = [e["input_ids"] for e in examples]

From 30668ee24cd935fac116a37a552b5c54b25465bd Mon Sep 17 00:00:00 2001
From: wlhgtc <hgtcwl@foxmail.com>
Date: Wed, 21 Oct 2020 07:50:15 +0800
Subject: [PATCH 10/31] Update examples/language-modeling/README.md

Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com>
---
 examples/language-modeling/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/language-modeling/README.md b/examples/language-modeling/README.md
index 6ebcfd802e33..e562437bef9d 100644
--- a/examples/language-modeling/README.md
+++ b/examples/language-modeling/README.md
@@ -45,7 +45,7 @@ slightly slower (over-fitting takes more epochs).
 
 We use the `--mlm` flag so that the script may change its loss function.
 
-If use whole-word masking, use both `--mlm` and `--wwm` flag(for English Model).
+If using whole-word masking, use both the`--mlm` and `--wwm` flags.
 
 ```bash
 export TRAIN_FILE=/path/to/dataset/wiki.train.raw

From d1c4d25f5ab90b091a9c64adf620ce2cc54fc00a Mon Sep 17 00:00:00 2001
From: wlhgtc <hgtcwl@foxmail.com>
Date: Wed, 21 Oct 2020 07:50:26 +0800
Subject: [PATCH 11/31] Update examples/language-modeling/README.md

Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com>
---
 examples/language-modeling/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/language-modeling/README.md b/examples/language-modeling/README.md
index e562437bef9d..fd214d0a420a 100644
--- a/examples/language-modeling/README.md
+++ b/examples/language-modeling/README.md
@@ -63,7 +63,7 @@ python run_language_modeling.py \
     --wwm
 ```
 
-For Chinese Model, we need to generate ref files, case it's char level.
+For Chinese models, we need to generate a reference files, case it's char level.
 
 ```bash
 export TRAIN_FILE=/path/to/dataset/wiki.train.raw

From 56bf4276f4288b524b6fc8bd0b98c612702d6b1f Mon Sep 17 00:00:00 2001
From: wlhgtc <hgtcwl@foxmail.com>
Date: Wed, 21 Oct 2020 07:50:40 +0800
Subject: [PATCH 12/31] Update
 examples/language-modeling/run_language_modeling.py

Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com>
---
 examples/language-modeling/run_language_modeling.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/language-modeling/run_language_modeling.py b/examples/language-modeling/run_language_modeling.py
index 77c0c3280059..19ccf9493e85 100644
--- a/examples/language-modeling/run_language_modeling.py
+++ b/examples/language-modeling/run_language_modeling.py
@@ -104,7 +104,7 @@ class DataTrainingArguments:
         metadata={"help": "An optional input evaluation data file to evaluate the perplexity on (a text file)."},
     )
     chinese_ref_file: Optional[str] = field(
-        default=None, metadata={"help": "An optional input ref data file for whole word mask(wwm) in Chinees."},
+        default=None, metadata={"help": "An optional input ref data file for whole word mask in Chinees."},
     )
     line_by_line: bool = field(
         default=False,

From 745e49dfdceea62c402f89adf4041e8b5a4cd0b7 Mon Sep 17 00:00:00 2001
From: wlhgtc <hgtcwl@foxmail.com>
Date: Wed, 21 Oct 2020 07:50:51 +0800
Subject: [PATCH 13/31] Update
 examples/language-modeling/run_language_modeling.py

Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com>
---
 examples/language-modeling/run_language_modeling.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/language-modeling/run_language_modeling.py b/examples/language-modeling/run_language_modeling.py
index 19ccf9493e85..6d294b691a10 100644
--- a/examples/language-modeling/run_language_modeling.py
+++ b/examples/language-modeling/run_language_modeling.py
@@ -114,7 +114,7 @@ class DataTrainingArguments:
     mlm: bool = field(
         default=False, metadata={"help": "Train with masked-language modeling loss instead of language modeling."}
     )
-    wwm: bool = field(default=False, metadata={"help": "Use Whole Word Mask."})
+    whole_word_mask: bool = field(default=False, metadata={"help": "Whether ot not to use whole word mask."})
     mlm_probability: float = field(
         default=0.15, metadata={"help": "Ratio of tokens to mask for masked language modeling loss"}
     )

From cf909c0779a00d31696d315c7bc2add62c6a7809 Mon Sep 17 00:00:00 2001
From: wlhgtc <hgtcwl@foxmail.com>
Date: Wed, 21 Oct 2020 07:51:01 +0800
Subject: [PATCH 14/31] Update
 examples/language-modeling/run_language_modeling.py

Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com>
---
 examples/language-modeling/run_language_modeling.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/language-modeling/run_language_modeling.py b/examples/language-modeling/run_language_modeling.py
index 6d294b691a10..56e2d2942b28 100644
--- a/examples/language-modeling/run_language_modeling.py
+++ b/examples/language-modeling/run_language_modeling.py
@@ -151,7 +151,7 @@ def _dataset(file_path):
         if args.line_by_line:
             if args.chinese_ref_file:
                 if not args.wwm or args.mlm:
-                    raise ValueError("Need set wwm and mlm to true for Chinese Whole Word Mask")
+                    raise ValueError("You need to set world whole masking and mlm to True for Chinese Whole Word Mask")
                 return LineByLineWithRefDataset(
                     tokenizer=tokenizer,
                     file_path=file_path,

From 960e4c8c76216b8263ab693f51d01cc94112019d Mon Sep 17 00:00:00 2001
From: wlhgtc <hgtcwl@foxmail.com>
Date: Wed, 21 Oct 2020 07:51:10 +0800
Subject: [PATCH 15/31] Update
 examples/language-modeling/run_language_modeling.py

Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com>
---
 examples/language-modeling/run_language_modeling.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/language-modeling/run_language_modeling.py b/examples/language-modeling/run_language_modeling.py
index 56e2d2942b28..fbb3d805da96 100644
--- a/examples/language-modeling/run_language_modeling.py
+++ b/examples/language-modeling/run_language_modeling.py
@@ -149,7 +149,7 @@ def get_dataset(
 ):
     def _dataset(file_path):
         if args.line_by_line:
-            if args.chinese_ref_file:
+            if args.chinese_ref_file is not None:
                 if not args.wwm or args.mlm:
                     raise ValueError("You need to set world whole masking and mlm to True for Chinese Whole Word Mask")
                 return LineByLineWithRefDataset(

From 45265b76f4ba547f505083954c214c7273a3c0ca Mon Sep 17 00:00:00 2001
From: Hgtc <hgtcwl@foxmail.com>
Date: Wed, 21 Oct 2020 08:42:15 +0800
Subject: [PATCH 16/31] change wwm to whole_word_mask

---
 examples/language-modeling/README.md                | 2 +-
 examples/language-modeling/run_language_modeling.py | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/examples/language-modeling/README.md b/examples/language-modeling/README.md
index fd214d0a420a..a22b63c44b19 100644
--- a/examples/language-modeling/README.md
+++ b/examples/language-modeling/README.md
@@ -63,7 +63,7 @@ python run_language_modeling.py \
     --wwm
 ```
 
-For Chinese models, we need to generate a reference files, case it's char level.
+For Chinese models, it's same with English model with only --mlm`. If using whole-word masking, we need to generate a reference files, case it's char level.
 
 ```bash
 export TRAIN_FILE=/path/to/dataset/wiki.train.raw
diff --git a/examples/language-modeling/run_language_modeling.py b/examples/language-modeling/run_language_modeling.py
index fbb3d805da96..b3873bb3ff48 100644
--- a/examples/language-modeling/run_language_modeling.py
+++ b/examples/language-modeling/run_language_modeling.py
@@ -150,7 +150,7 @@ def get_dataset(
     def _dataset(file_path):
         if args.line_by_line:
             if args.chinese_ref_file is not None:
-                if not args.wwm or args.mlm:
+                if not args.whole_word_mask or not args.mlm:
                     raise ValueError("You need to set world whole masking and mlm to True for Chinese Whole Word Mask")
                 return LineByLineWithRefDataset(
                     tokenizer=tokenizer,
@@ -283,7 +283,7 @@ def main():
             tokenizer=tokenizer, plm_probability=data_args.plm_probability, max_span_length=data_args.max_span_length,
         )
     else:
-        if data_args.mlm and data_args.wwm:
+        if data_args.mlm and data_args.whole_word_mask:
             data_collator = DataCollatorForWholeWordMask(
                 tokenizer=tokenizer, mlm_probability=data_args.mlm_probability
             )

From 68d883250615622e6b112a0b5faab3caf44b8de6 Mon Sep 17 00:00:00 2001
From: Hgtc <hgtcwl@foxmail.com>
Date: Wed, 21 Oct 2020 08:43:04 +0800
Subject: [PATCH 17/31] reformat code

---
 examples/language-modeling/run_language_modeling.py |  7 +++++--
 src/transformers/data/datasets/language_modeling.py | 13 +++++++++++--
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/examples/language-modeling/run_language_modeling.py b/examples/language-modeling/run_language_modeling.py
index b3873bb3ff48..3ba34071a95f 100644
--- a/examples/language-modeling/run_language_modeling.py
+++ b/examples/language-modeling/run_language_modeling.py
@@ -104,7 +104,8 @@ class DataTrainingArguments:
         metadata={"help": "An optional input evaluation data file to evaluate the perplexity on (a text file)."},
     )
     chinese_ref_file: Optional[str] = field(
-        default=None, metadata={"help": "An optional input ref data file for whole word mask in Chinees."},
+        default=None,
+        metadata={"help": "An optional input ref data file for whole word mask in Chinees."},
     )
     line_by_line: bool = field(
         default=False,
@@ -280,7 +281,9 @@ def main():
     )
     if config.model_type == "xlnet":
         data_collator = DataCollatorForPermutationLanguageModeling(
-            tokenizer=tokenizer, plm_probability=data_args.plm_probability, max_span_length=data_args.max_span_length,
+            tokenizer=tokenizer,
+            plm_probability=data_args.plm_probability,
+            max_span_length=data_args.max_span_length,
         )
     else:
         if data_args.mlm and data_args.whole_word_mask:
diff --git a/src/transformers/data/datasets/language_modeling.py b/src/transformers/data/datasets/language_modeling.py
index 5b81466cb983..0633c2c570b2 100644
--- a/src/transformers/data/datasets/language_modeling.py
+++ b/src/transformers/data/datasets/language_modeling.py
@@ -37,7 +37,11 @@ def __init__(
         directory, filename = os.path.split(file_path)
         cached_features_file = os.path.join(
             cache_dir if cache_dir is not None else directory,
-            "cached_lm_{}_{}_{}".format(tokenizer.__class__.__name__, str(block_size), filename,),
+            "cached_lm_{}_{}_{}".format(
+                tokenizer.__class__.__name__,
+                str(block_size),
+                filename,
+            ),
         )
 
         # Make sure only the first process in distributed training processes the dataset,
@@ -310,7 +314,12 @@ def __init__(
 
         directory, filename = os.path.split(file_path)
         cached_features_file = os.path.join(
-            directory, "cached_nsp_{}_{}_{}".format(tokenizer.__class__.__name__, str(block_size), filename,),
+            directory,
+            "cached_nsp_{}_{}_{}".format(
+                tokenizer.__class__.__name__,
+                str(block_size),
+                filename,
+            ),
         )
 
         self.tokenizer = tokenizer

From ea21325a654d466c227d37e80f55c00239828a4e Mon Sep 17 00:00:00 2001
From: Hgtc <hgtcwl@foxmail.com>
Date: Wed, 21 Oct 2020 10:20:08 +0800
Subject: [PATCH 18/31] reformat

---
 examples/language-modeling/chinese_ref.py           | 7 +++----
 examples/language-modeling/run_language_modeling.py | 4 ++--
 src/transformers/data/data_collator.py              | 6 ++++--
 src/transformers/data/datasets/language_modeling.py | 3 ++-
 4 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/examples/language-modeling/chinese_ref.py b/examples/language-modeling/chinese_ref.py
index b84ab02b42fd..02a1038f1fe6 100644
--- a/examples/language-modeling/chinese_ref.py
+++ b/examples/language-modeling/chinese_ref.py
@@ -1,11 +1,10 @@
-import json
-import random
 import argparse
+import json
+from typing import List
+
 from ltp import LTP
 from transformers.tokenization_bert import BertTokenizer
 
-from typing import List
-
 
 def _is_chinese_char(cp):
     """Checks whether CP is the codepoint of a CJK character."""
diff --git a/examples/language-modeling/run_language_modeling.py b/examples/language-modeling/run_language_modeling.py
index 3ba34071a95f..4b429f938e23 100644
--- a/examples/language-modeling/run_language_modeling.py
+++ b/examples/language-modeling/run_language_modeling.py
@@ -23,12 +23,12 @@
 import logging
 import math
 import os
-from dataclasses import dataclass, field
 from glob import glob
 from typing import Optional
 
 from torch.utils.data import ConcatDataset
 
+from dataclasses import dataclass, field
 from transformers import (
     CONFIG_MAPPING,
     MODEL_WITH_LM_HEAD_MAPPING,
@@ -36,8 +36,8 @@
     AutoModelWithLMHead,
     AutoTokenizer,
     DataCollatorForLanguageModeling,
-    DataCollatorForWholeWordMask,
     DataCollatorForPermutationLanguageModeling,
+    DataCollatorForWholeWordMask,
     HfArgumentParser,
     LineByLineTextDataset,
     LineByLineWithRefDataset,
diff --git a/src/transformers/data/data_collator.py b/src/transformers/data/data_collator.py
index 0b9d2b9c3701..7b5796796af5 100644
--- a/src/transformers/data/data_collator.py
+++ b/src/transformers/data/data_collator.py
@@ -1,9 +1,11 @@
-from dataclasses import dataclass
-from typing import Any, Callable, Dict, List, NewType, Optional, Tuple, Union
 import random
+from typing import Any, Callable, Dict, List, NewType, Optional, Tuple, Union
+
 import torch
 from torch.nn.utils.rnn import pad_sequence
 
+from dataclasses import dataclass
+
 from ..tokenization_utils_base import BatchEncoding, PaddingStrategy, PreTrainedTokenizerBase
 
 
diff --git a/src/transformers/data/datasets/language_modeling.py b/src/transformers/data/datasets/language_modeling.py
index 0633c2c570b2..9cd337f1edd2 100644
--- a/src/transformers/data/datasets/language_modeling.py
+++ b/src/transformers/data/datasets/language_modeling.py
@@ -1,9 +1,10 @@
+import json
 import os
 import pickle
 import random
 import time
 from typing import Dict, List, Optional
-import json
+
 import torch
 from torch.utils.data.dataset import Dataset
 

From bf976a4732d05d29d1cc60a9e1f0f2fc4cfe00d7 Mon Sep 17 00:00:00 2001
From: Hgtc <hgtcwl@foxmail.com>
Date: Wed, 21 Oct 2020 15:00:12 +0800
Subject: [PATCH 19/31] format

---
 examples/language-modeling/run_language_modeling.py | 2 +-
 src/transformers/__init__.py                        | 2 +-
 src/transformers/data/data_collator.py              | 3 +--
 3 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/examples/language-modeling/run_language_modeling.py b/examples/language-modeling/run_language_modeling.py
index 4b429f938e23..6a3f3f63f700 100644
--- a/examples/language-modeling/run_language_modeling.py
+++ b/examples/language-modeling/run_language_modeling.py
@@ -23,12 +23,12 @@
 import logging
 import math
 import os
+from dataclasses import dataclass, field
 from glob import glob
 from typing import Optional
 
 from torch.utils.data import ConcatDataset
 
-from dataclasses import dataclass, field
 from transformers import (
     CONFIG_MAPPING,
     MODEL_WITH_LM_HEAD_MAPPING,
diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py
index 8537cb431185..6bfb8a9c6b7c 100755
--- a/src/transformers/__init__.py
+++ b/src/transformers/__init__.py
@@ -281,10 +281,10 @@
     from .data.data_collator import (
         DataCollator,
         DataCollatorForLanguageModeling,
-        DataCollatorForWholeWordMask,
         DataCollatorForNextSentencePrediction,
         DataCollatorForPermutationLanguageModeling,
         DataCollatorForSOP,
+        DataCollatorForWholeWordMask,
         DataCollatorWithPadding,
         default_data_collator,
     )
diff --git a/src/transformers/data/data_collator.py b/src/transformers/data/data_collator.py
index 7b5796796af5..d05061a7c020 100644
--- a/src/transformers/data/data_collator.py
+++ b/src/transformers/data/data_collator.py
@@ -1,11 +1,10 @@
 import random
+from dataclasses import dataclass
 from typing import Any, Callable, Dict, List, NewType, Optional, Tuple, Union
 
 import torch
 from torch.nn.utils.rnn import pad_sequence
 
-from dataclasses import dataclass
-
 from ..tokenization_utils_base import BatchEncoding, PaddingStrategy, PreTrainedTokenizerBase
 
 

From 9b50ee3ad659d3827ff2331419e1edcc35816ec7 Mon Sep 17 00:00:00 2001
From: Sylvain Gugger <sylvain.gugger@gmail.com>
Date: Wed, 21 Oct 2020 11:08:54 -0400
Subject: [PATCH 20/31] Code quality

---
 src/transformers/utils/dummy_pt_objects.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/transformers/utils/dummy_pt_objects.py b/src/transformers/utils/dummy_pt_objects.py
index 3e152be24046..9e4a8ad6f7b8 100644
--- a/src/transformers/utils/dummy_pt_objects.py
+++ b/src/transformers/utils/dummy_pt_objects.py
@@ -45,6 +45,11 @@ def __init__(self, *args, **kwargs):
         requires_pytorch(self)
 
 
+class DataCollatorForWholeWordMask:
+    def __init__(self, *args, **kwargs):
+        requires_pytorch(self)
+
+
 class DataCollatorWithPadding:
     def __init__(self, *args, **kwargs):
         requires_pytorch(self)
@@ -69,6 +74,11 @@ def __init__(self, *args, **kwargs):
         requires_pytorch(self)
 
 
+class LineByLineWithRefDataset:
+    def __init__(self, *args, **kwargs):
+        requires_pytorch(self)
+
+
 class LineByLineWithSOPTextDataset:
     def __init__(self, *args, **kwargs):
         requires_pytorch(self)

From a285531d1b0a3739c9c0307ed21536df2ae12972 Mon Sep 17 00:00:00 2001
From: Hgtc <hgtcwl@foxmail.com>
Date: Thu, 22 Oct 2020 18:04:29 +0800
Subject: [PATCH 21/31] ADD: update chinese ref readme

---
 examples/language-modeling/README.md | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/examples/language-modeling/README.md b/examples/language-modeling/README.md
index a22b63c44b19..26797f635f78 100644
--- a/examples/language-modeling/README.md
+++ b/examples/language-modeling/README.md
@@ -65,6 +65,17 @@ python run_language_modeling.py \
 
 For Chinese models, it's same with English model with only --mlm`. If using whole-word masking, we need to generate a reference files, case it's char level.
 
+**Q :** Why ref file ?
+
+**A :** Suppose we have a Chinese sentence like : `我喜欢你。` The original Chinese-BERT will tokenize it as `['我','喜','欢','你']` in char level.
+Actually, `喜欢` is a whole word. For whole word mask proxy, We need res like `['我','喜','##欢','你']`.
+So we need a ref file to tell model which pos of BERT original token should be added `##`.
+
+**Q :** Why LTP ?
+
+**A :** Cause the best known Chinese WWM BERT is [https://github.com/ymcui/Chinese-BERT-wwm](https://github.com/ymcui/Chinese-BERT-wwm). It works well on so many Chines Task like CLUE (Chinese GLUE).
+They use LTP, so if we want to fine-tune their model, we need LTP.
+
 ```bash
 export TRAIN_FILE=/path/to/dataset/wiki.train.raw
 export LTP_RESOURCE=/path/to/ltp/tokenizer

From ed15eba3d772eaaeb2d48a725633b3581eeb88a6 Mon Sep 17 00:00:00 2001
From: Hgtc <hgtcwl@foxmail.com>
Date: Thu, 22 Oct 2020 18:07:43 +0800
Subject: [PATCH 22/31] MOD: small changes

---
 examples/language-modeling/README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/examples/language-modeling/README.md b/examples/language-modeling/README.md
index 26797f635f78..fc7410ae10fc 100644
--- a/examples/language-modeling/README.md
+++ b/examples/language-modeling/README.md
@@ -67,13 +67,13 @@ For Chinese models, it's same with English model with only --mlm`. If using whol
 
 **Q :** Why ref file ?
 
-**A :** Suppose we have a Chinese sentence like : `我喜欢你。` The original Chinese-BERT will tokenize it as `['我','喜','欢','你']` in char level.
+**A :** Suppose we have a Chinese sentence like : `我喜欢你` The original Chinese-BERT will tokenize it as `['我','喜','欢','你']` in char level.
 Actually, `喜欢` is a whole word. For whole word mask proxy, We need res like `['我','喜','##欢','你']`.
 So we need a ref file to tell model which pos of BERT original token should be added `##`.
 
 **Q :** Why LTP ?
 
-**A :** Cause the best known Chinese WWM BERT is [https://github.com/ymcui/Chinese-BERT-wwm](https://github.com/ymcui/Chinese-BERT-wwm). It works well on so many Chines Task like CLUE (Chinese GLUE).
+**A :** Cause the best known Chinese WWM BERT is [Chinese-BERT-wwm](https://github.com/ymcui/Chinese-BERT-wwm). It works well on so many Chines Task like CLUE (Chinese GLUE).
 They use LTP, so if we want to fine-tune their model, we need LTP.
 
 ```bash

From 87ab48c46349fc86c604f91a5b7e9740a3dbeb4f Mon Sep 17 00:00:00 2001
From: Hgtc <hgtcwl@foxmail.com>
Date: Thu, 22 Oct 2020 18:07:56 +0800
Subject: [PATCH 23/31] MOD: small changes2

---
 examples/language-modeling/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/language-modeling/README.md b/examples/language-modeling/README.md
index fc7410ae10fc..97e6e708fdf4 100644
--- a/examples/language-modeling/README.md
+++ b/examples/language-modeling/README.md
@@ -73,7 +73,7 @@ So we need a ref file to tell model which pos of BERT original token should be a
 
 **Q :** Why LTP ?
 
-**A :** Cause the best known Chinese WWM BERT is [Chinese-BERT-wwm](https://github.com/ymcui/Chinese-BERT-wwm). It works well on so many Chines Task like CLUE (Chinese GLUE).
+**A :** Cause the best known Chinese WWM BERT is [Chinese-BERT-wwm](https://github.com/ymcui/Chinese-BERT-wwm) by HIT. It works well on so many Chines Task like CLUE (Chinese GLUE).
 They use LTP, so if we want to fine-tune their model, we need LTP.
 
 ```bash

From beeb7aa49546fec1e9b8244689dc85cdf0cf6fad Mon Sep 17 00:00:00 2001
From: Hgtc <hgtcwl@foxmail.com>
Date: Thu, 22 Oct 2020 18:29:57 +0800
Subject: [PATCH 24/31] update readme

---
 examples/language-modeling/README.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/examples/language-modeling/README.md b/examples/language-modeling/README.md
index 97e6e708fdf4..80034959221f 100644
--- a/examples/language-modeling/README.md
+++ b/examples/language-modeling/README.md
@@ -88,7 +88,7 @@ python chinese_ref.py \
     --bert=$BERT_RESOURCE \
     --save_path=$SAVE_PATH 
 ```
-Then: 
+Now Chinese Ref is only supported by `LineByLineWithRefDataset` Class, so we need add `line_by_line` flag: 
 
 
 ```bash
@@ -106,6 +106,7 @@ python run_language_modeling.py \
     --do_eval \
     --eval_data_file=$TEST_FILE \
     --mlm \
+    --line_by_line \
     --wwm
 ```
 

From 0daafa534017bc8f35566ec316748d157506c9b4 Mon Sep 17 00:00:00 2001
From: Hgtc <hgtcwl@foxmail.com>
Date: Wed, 28 Oct 2020 18:08:47 +0800
Subject: [PATCH 25/31] fix eval ref file miss bug

---
 examples/language-modeling/README.md           |  7 +++++--
 .../language-modeling/run_language_modeling.py | 18 +++++++++++-------
 2 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/examples/language-modeling/README.md b/examples/language-modeling/README.md
index d5c0e619dcce..06fcafe565a3 100644
--- a/examples/language-modeling/README.md
+++ b/examples/language-modeling/README.md
@@ -63,7 +63,7 @@ python run_language_modeling.py \
     --whole_word_mask
 ```
 
-For Chinese models, it's same with English model with only --mlm`. If using whole-word masking, we need to generate a reference files, case it's char level.
+For Chinese models, it's same with English model with only `--mlm`. If using whole-word masking, we need to generate a reference files, cause it's char level.
 
 **Q :** Why ref file ?
 
@@ -76,6 +76,9 @@ So we need a ref file to tell model which pos of BERT original token should be a
 **A :** Cause the best known Chinese WWM BERT is [Chinese-BERT-wwm](https://github.com/ymcui/Chinese-BERT-wwm) by HIT. It works well on so many Chines Task like CLUE (Chinese GLUE).
 They use LTP, so if we want to fine-tune their model, we need LTP.
 
+Now LTP only works well on `transformers==3.2.0`. So we don't add it to requirements.txt.
+
+You have to run `chinese_ref.py` on older version, but `run_language_modeling.py` on newest version. 
 ```bash
 export TRAIN_FILE=/path/to/dataset/wiki.train.raw
 export LTP_RESOURCE=/path/to/ltp/tokenizer
@@ -84,7 +87,7 @@ export SAVE_PATH=/path/to/data/ref.txt
 
 python chinese_ref.py \
     --file_name=$TRAIN_FILE \
-    --ltp=$LTP_RESOURCE
+    --ltp=$LTP_RESOURCE \
     --bert=$BERT_RESOURCE \
     --save_path=$SAVE_PATH 
 ```
diff --git a/examples/language-modeling/run_language_modeling.py b/examples/language-modeling/run_language_modeling.py
index 6a3f3f63f700..4ab0319f68ce 100644
--- a/examples/language-modeling/run_language_modeling.py
+++ b/examples/language-modeling/run_language_modeling.py
@@ -103,9 +103,13 @@ class DataTrainingArguments:
         default=None,
         metadata={"help": "An optional input evaluation data file to evaluate the perplexity on (a text file)."},
     )
-    chinese_ref_file: Optional[str] = field(
+    train_ref_file: Optional[str] = field(
         default=None,
-        metadata={"help": "An optional input ref data file for whole word mask in Chinees."},
+        metadata={"help": "An optional input train ref data file for whole word mask in Chinese."},
+    )
+    eval_ref_file: Optional[str] = field(
+        default=None,
+        metadata={"help": "An optional input eval ref data file for whole word mask in Chinese."},
     )
     line_by_line: bool = field(
         default=False,
@@ -148,16 +152,16 @@ def get_dataset(
     evaluate: bool = False,
     cache_dir: Optional[str] = None,
 ):
-    def _dataset(file_path):
+    def _dataset(file_path, ref_path = None):
         if args.line_by_line:
-            if args.chinese_ref_file is not None:
+            if ref_path is not None:
                 if not args.whole_word_mask or not args.mlm:
                     raise ValueError("You need to set world whole masking and mlm to True for Chinese Whole Word Mask")
                 return LineByLineWithRefDataset(
                     tokenizer=tokenizer,
                     file_path=file_path,
                     block_size=args.block_size,
-                    ref_path=args.chinese_ref_file,
+                    ref_path=ref_path,
                 )
 
             return LineByLineTextDataset(tokenizer=tokenizer, file_path=file_path, block_size=args.block_size)
@@ -171,11 +175,11 @@ def _dataset(file_path):
             )
 
     if evaluate:
-        return _dataset(args.eval_data_file)
+        return _dataset(args.eval_data_file, args.eval_ref_file)
     elif args.train_data_files:
         return ConcatDataset([_dataset(f) for f in glob(args.train_data_files)])
     else:
-        return _dataset(args.train_data_file)
+        return _dataset(args.train_data_file, args.train_ref_file)
 
 
 def main():

From 3c9413c76c43c3b560221db719ea57f04e34579e Mon Sep 17 00:00:00 2001
From: Hgtc <hgtcwl@foxmail.com>
Date: Wed, 28 Oct 2020 18:24:00 +0800
Subject: [PATCH 26/31] format file

---
 .../language-modeling/run_language_modeling.py  | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)

diff --git a/examples/language-modeling/run_language_modeling.py b/examples/language-modeling/run_language_modeling.py
index 4ab0319f68ce..26066dde197d 100644
--- a/examples/language-modeling/run_language_modeling.py
+++ b/examples/language-modeling/run_language_modeling.py
@@ -104,12 +104,10 @@ class DataTrainingArguments:
         metadata={"help": "An optional input evaluation data file to evaluate the perplexity on (a text file)."},
     )
     train_ref_file: Optional[str] = field(
-        default=None,
-        metadata={"help": "An optional input train ref data file for whole word mask in Chinese."},
+        default=None, metadata={"help": "An optional input train ref data file for whole word mask in Chinese."},
     )
     eval_ref_file: Optional[str] = field(
-        default=None,
-        metadata={"help": "An optional input eval ref data file for whole word mask in Chinese."},
+        default=None, metadata={"help": "An optional input eval ref data file for whole word mask in Chinese."},
     )
     line_by_line: bool = field(
         default=False,
@@ -152,16 +150,13 @@ def get_dataset(
     evaluate: bool = False,
     cache_dir: Optional[str] = None,
 ):
-    def _dataset(file_path, ref_path = None):
+    def _dataset(file_path, ref_path=None):
         if args.line_by_line:
             if ref_path is not None:
                 if not args.whole_word_mask or not args.mlm:
                     raise ValueError("You need to set world whole masking and mlm to True for Chinese Whole Word Mask")
                 return LineByLineWithRefDataset(
-                    tokenizer=tokenizer,
-                    file_path=file_path,
-                    block_size=args.block_size,
-                    ref_path=ref_path,
+                    tokenizer=tokenizer, file_path=file_path, block_size=args.block_size, ref_path=ref_path,
                 )
 
             return LineByLineTextDataset(tokenizer=tokenizer, file_path=file_path, block_size=args.block_size)
@@ -285,9 +280,7 @@ def main():
     )
     if config.model_type == "xlnet":
         data_collator = DataCollatorForPermutationLanguageModeling(
-            tokenizer=tokenizer,
-            plm_probability=data_args.plm_probability,
-            max_span_length=data_args.max_span_length,
+            tokenizer=tokenizer, plm_probability=data_args.plm_probability, max_span_length=data_args.max_span_length,
         )
     else:
         if data_args.mlm and data_args.whole_word_mask:

From e6fecbb3fe82a41d01b945bc81ddebc6da930789 Mon Sep 17 00:00:00 2001
From: Hgtc <hgtcwl@foxmail.com>
Date: Thu, 29 Oct 2020 10:57:44 +0800
Subject: [PATCH 27/31] MOD: move ref code to contrib

---
 .../chinese_ref.py => contrib/run_chinese_ref.py} |  0
 examples/language-modeling/README.md              |  7 ++++---
 .../language-modeling/run_language_modeling.py    | 15 +++++++++++----
 3 files changed, 15 insertions(+), 7 deletions(-)
 rename examples/{language-modeling/chinese_ref.py => contrib/run_chinese_ref.py} (100%)

diff --git a/examples/language-modeling/chinese_ref.py b/examples/contrib/run_chinese_ref.py
similarity index 100%
rename from examples/language-modeling/chinese_ref.py
rename to examples/contrib/run_chinese_ref.py
diff --git a/examples/language-modeling/README.md b/examples/language-modeling/README.md
index 06fcafe565a3..6e6dc699ae09 100644
--- a/examples/language-modeling/README.md
+++ b/examples/language-modeling/README.md
@@ -76,16 +76,17 @@ So we need a ref file to tell model which pos of BERT original token should be a
 **A :** Cause the best known Chinese WWM BERT is [Chinese-BERT-wwm](https://github.com/ymcui/Chinese-BERT-wwm) by HIT. It works well on so many Chines Task like CLUE (Chinese GLUE).
 They use LTP, so if we want to fine-tune their model, we need LTP.
 
-Now LTP only works well on `transformers==3.2.0`. So we don't add it to requirements.txt.
+Now LTP only only works well on `transformers==3.2.0`. So we don't add it to requirements.txt.
+You need to check to `3.2.0` for `run_chinese_ref.py`. And the code  could be found in `examples/contrib`.
+
 
-You have to run `chinese_ref.py` on older version, but `run_language_modeling.py` on newest version. 
 ```bash
 export TRAIN_FILE=/path/to/dataset/wiki.train.raw
 export LTP_RESOURCE=/path/to/ltp/tokenizer
 export BERT_RESOURCE=/path/to/bert/tokenizer
 export SAVE_PATH=/path/to/data/ref.txt
 
-python chinese_ref.py \
+python examples/contrib/run_chinese_ref.py \
     --file_name=$TRAIN_FILE \
     --ltp=$LTP_RESOURCE \
     --bert=$BERT_RESOURCE \
diff --git a/examples/language-modeling/run_language_modeling.py b/examples/language-modeling/run_language_modeling.py
index 26066dde197d..e1b6ff0c960b 100644
--- a/examples/language-modeling/run_language_modeling.py
+++ b/examples/language-modeling/run_language_modeling.py
@@ -104,10 +104,12 @@ class DataTrainingArguments:
         metadata={"help": "An optional input evaluation data file to evaluate the perplexity on (a text file)."},
     )
     train_ref_file: Optional[str] = field(
-        default=None, metadata={"help": "An optional input train ref data file for whole word mask in Chinese."},
+        default=None,
+        metadata={"help": "An optional input train ref data file for whole word mask in Chinese."},
     )
     eval_ref_file: Optional[str] = field(
-        default=None, metadata={"help": "An optional input eval ref data file for whole word mask in Chinese."},
+        default=None,
+        metadata={"help": "An optional input eval ref data file for whole word mask in Chinese."},
     )
     line_by_line: bool = field(
         default=False,
@@ -156,7 +158,10 @@ def _dataset(file_path, ref_path=None):
                 if not args.whole_word_mask or not args.mlm:
                     raise ValueError("You need to set world whole masking and mlm to True for Chinese Whole Word Mask")
                 return LineByLineWithRefDataset(
-                    tokenizer=tokenizer, file_path=file_path, block_size=args.block_size, ref_path=ref_path,
+                    tokenizer=tokenizer,
+                    file_path=file_path,
+                    block_size=args.block_size,
+                    ref_path=ref_path,
                 )
 
             return LineByLineTextDataset(tokenizer=tokenizer, file_path=file_path, block_size=args.block_size)
@@ -280,7 +285,9 @@ def main():
     )
     if config.model_type == "xlnet":
         data_collator = DataCollatorForPermutationLanguageModeling(
-            tokenizer=tokenizer, plm_probability=data_args.plm_probability, max_span_length=data_args.max_span_length,
+            tokenizer=tokenizer,
+            plm_probability=data_args.plm_probability,
+            max_span_length=data_args.max_span_length,
         )
     else:
         if data_args.mlm and data_args.whole_word_mask:

From 02cadaec303f55a39ddd8fa9f4bde533a347b107 Mon Sep 17 00:00:00 2001
From: Hgtc <hgtcwl@foxmail.com>
Date: Thu, 29 Oct 2020 19:07:15 +0800
Subject: [PATCH 28/31] MOD: add delimeter check

---
 examples/contrib/run_chinese_ref.py | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/examples/contrib/run_chinese_ref.py b/examples/contrib/run_chinese_ref.py
index 02a1038f1fe6..7c31eef25c36 100644
--- a/examples/contrib/run_chinese_ref.py
+++ b/examples/contrib/run_chinese_ref.py
@@ -17,14 +17,14 @@ def _is_chinese_char(cp):
     # space-separated words, so they are not treated specially and handled
     # like the all of the other languages.
     if (
-        (cp >= 0x4E00 and cp <= 0x9FFF)
-        or (cp >= 0x3400 and cp <= 0x4DBF)  #
-        or (cp >= 0x20000 and cp <= 0x2A6DF)  #
-        or (cp >= 0x2A700 and cp <= 0x2B73F)  #
-        or (cp >= 0x2B740 and cp <= 0x2B81F)  #
-        or (cp >= 0x2B820 and cp <= 0x2CEAF)  #
-        or (cp >= 0xF900 and cp <= 0xFAFF)
-        or (cp >= 0x2F800 and cp <= 0x2FA1F)  #
+            (cp >= 0x4E00 and cp <= 0x9FFF)
+            or (cp >= 0x3400 and cp <= 0x4DBF)  #
+            or (cp >= 0x20000 and cp <= 0x2A6DF)  #
+            or (cp >= 0x2A700 and cp <= 0x2B73F)  #
+            or (cp >= 0x2B740 and cp <= 0x2B81F)  #
+            or (cp >= 0x2B820 and cp <= 0x2CEAF)  #
+            or (cp >= 0xF900 and cp <= 0xFAFF)
+            or (cp >= 0x2F800 and cp <= 0x2FA1F)  #
     ):  #
         return True
 
@@ -63,7 +63,7 @@ def add_sub_symbol(bert_tokens: List[str], chinese_word_set: set()):
         if is_chinese(bert_word[start]):
             l = min(end - start, max_word_len)
             for i in range(l, 1, -1):
-                whole_word = "".join(bert_word[start : start + i])
+                whole_word = "".join(bert_word[start: start + i])
                 if whole_word in chinese_word_set:
                     for j in range(start + 1, start + i):
                         bert_word[j] = "##" + bert_word[j]
@@ -79,14 +79,14 @@ def prepare_ref(lines: List[str], ltp_tokenizer: LTP, bert_tokenizer: BertTokeni
     ltp_res = []
 
     for i in range(0, len(lines), 100):
-        res = ltp_tokenizer.seg(lines[i : i + 100])[0]
+        res = ltp_tokenizer.seg(lines[i: i + 100])[0]
         res = [get_chinese_word(r) for r in res]
         ltp_res.extend(res)
     assert len(ltp_res) == len(lines)
 
     bert_res = []
     for i in range(0, len(lines), 100):
-        res = bert_tokenizer(lines[i : i + 100], add_special_tokens=True, truncation=True, max_length=512)
+        res = bert_tokenizer(lines[i: i + 100], add_special_tokens=True, truncation=True, max_length=512)
         bert_res.extend(res["input_ids"])
     assert len(bert_res) == len(lines)
 
@@ -118,7 +118,7 @@ def main(args):
     # If we want to fine-tune these model, we have to use same tokenizer : LTP (https://github.com/HIT-SCIR/ltp)
     with open(args.file_name, "r", encoding="utf-8") as f:
         data = f.readlines()
-
+    data = [line.strip() for line in data if len(line) > 0 and not line.isspace()]  # avoid delimiter like '\u2029'
     ltp_tokenizer = LTP(args.ltp)  # faster in GPU device
     bert_tokenizer = BertTokenizer.from_pretrained(args.bert)
 

From 120a1e4d2fc16d3af3975751000979ed8dd6f680 Mon Sep 17 00:00:00 2001
From: Hgtc <hgtcwl@foxmail.com>
Date: Thu, 29 Oct 2020 19:09:24 +0800
Subject: [PATCH 29/31] reformat code

---
 examples/contrib/run_chinese_ref.py           | 22 +++++++++----------
 .../data/datasets/language_modeling.py        | 13 ++++++-----
 2 files changed, 18 insertions(+), 17 deletions(-)

diff --git a/examples/contrib/run_chinese_ref.py b/examples/contrib/run_chinese_ref.py
index 7c31eef25c36..8ec7b7bc5057 100644
--- a/examples/contrib/run_chinese_ref.py
+++ b/examples/contrib/run_chinese_ref.py
@@ -17,14 +17,14 @@ def _is_chinese_char(cp):
     # space-separated words, so they are not treated specially and handled
     # like the all of the other languages.
     if (
-            (cp >= 0x4E00 and cp <= 0x9FFF)
-            or (cp >= 0x3400 and cp <= 0x4DBF)  #
-            or (cp >= 0x20000 and cp <= 0x2A6DF)  #
-            or (cp >= 0x2A700 and cp <= 0x2B73F)  #
-            or (cp >= 0x2B740 and cp <= 0x2B81F)  #
-            or (cp >= 0x2B820 and cp <= 0x2CEAF)  #
-            or (cp >= 0xF900 and cp <= 0xFAFF)
-            or (cp >= 0x2F800 and cp <= 0x2FA1F)  #
+        (cp >= 0x4E00 and cp <= 0x9FFF)
+        or (cp >= 0x3400 and cp <= 0x4DBF)  #
+        or (cp >= 0x20000 and cp <= 0x2A6DF)  #
+        or (cp >= 0x2A700 and cp <= 0x2B73F)  #
+        or (cp >= 0x2B740 and cp <= 0x2B81F)  #
+        or (cp >= 0x2B820 and cp <= 0x2CEAF)  #
+        or (cp >= 0xF900 and cp <= 0xFAFF)
+        or (cp >= 0x2F800 and cp <= 0x2FA1F)  #
     ):  #
         return True
 
@@ -63,7 +63,7 @@ def add_sub_symbol(bert_tokens: List[str], chinese_word_set: set()):
         if is_chinese(bert_word[start]):
             l = min(end - start, max_word_len)
             for i in range(l, 1, -1):
-                whole_word = "".join(bert_word[start: start + i])
+                whole_word = "".join(bert_word[start : start + i])
                 if whole_word in chinese_word_set:
                     for j in range(start + 1, start + i):
                         bert_word[j] = "##" + bert_word[j]
@@ -79,14 +79,14 @@ def prepare_ref(lines: List[str], ltp_tokenizer: LTP, bert_tokenizer: BertTokeni
     ltp_res = []
 
     for i in range(0, len(lines), 100):
-        res = ltp_tokenizer.seg(lines[i: i + 100])[0]
+        res = ltp_tokenizer.seg(lines[i : i + 100])[0]
         res = [get_chinese_word(r) for r in res]
         ltp_res.extend(res)
     assert len(ltp_res) == len(lines)
 
     bert_res = []
     for i in range(0, len(lines), 100):
-        res = bert_tokenizer(lines[i: i + 100], add_special_tokens=True, truncation=True, max_length=512)
+        res = bert_tokenizer(lines[i : i + 100], add_special_tokens=True, truncation=True, max_length=512)
         bert_res.extend(res["input_ids"])
     assert len(bert_res) == len(lines)
 
diff --git a/src/transformers/data/datasets/language_modeling.py b/src/transformers/data/datasets/language_modeling.py
index 8aa72c955f1a..d50b81f861f2 100644
--- a/src/transformers/data/datasets/language_modeling.py
+++ b/src/transformers/data/datasets/language_modeling.py
@@ -13,7 +13,6 @@
 from ...tokenization_utils import PreTrainedTokenizer
 from ...utils import logging
 
-
 logger = logging.get_logger(__name__)
 
 
@@ -128,15 +127,17 @@ def __init__(self, tokenizer: PreTrainedTokenizer, file_path: str, block_size: i
         logger.info("Creating features from dataset file at %s", file_path)
         logger.info("Use ref segment results at %s", ref_path)
         with open(file_path, encoding="utf-8") as f:
-            data = [line for line in f.read().splitlines() if (len(line) > 0 and not line.isspace())]
-        batch_encoding = tokenizer(data, add_special_tokens=True, truncation=True, max_length=block_size)
-        self.examples = batch_encoding["input_ids"]
-        self.examples = [{"input_ids": torch.tensor(e, dtype=torch.long)} for e in self.examples]
-
+            data = f.readlines()  # use this method to avoid delimiter '\u2029' to split a line
+        data = [line.strip() for line in data if len(line) > 0 and not line.isspace()]
         # Get ref inf from file
         with open(ref_path, encoding="utf-8") as f:
             ref = [json.loads(line) for line in f.read().splitlines() if (len(line) > 0 and not line.isspace())]
         assert len(data) == len(ref)
+
+        batch_encoding = tokenizer(data, add_special_tokens=True, truncation=True, max_length=block_size)
+        self.examples = batch_encoding["input_ids"]
+        self.examples = [{"input_ids": torch.tensor(e, dtype=torch.long)} for e in self.examples]
+
         n = len(self.examples)
         for i in range(n):
             self.examples[i]["chinese_ref"] = torch.tensor(ref[i], dtype=torch.long)

From f299194b6d541c019ddc68f741346dc504950cff Mon Sep 17 00:00:00 2001
From: Hgtc <hgtcwl@foxmail.com>
Date: Thu, 29 Oct 2020 19:20:29 +0800
Subject: [PATCH 30/31] refomat code

---
 src/transformers/data/datasets/language_modeling.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/transformers/data/datasets/language_modeling.py b/src/transformers/data/datasets/language_modeling.py
index d50b81f861f2..221b541d6350 100644
--- a/src/transformers/data/datasets/language_modeling.py
+++ b/src/transformers/data/datasets/language_modeling.py
@@ -13,6 +13,7 @@
 from ...tokenization_utils import PreTrainedTokenizer
 from ...utils import logging
 
+
 logger = logging.get_logger(__name__)
 
 

From de61598e58e85b059f2d0d23c66d6c9c8c0af1e9 Mon Sep 17 00:00:00 2001
From: Lysandre Debut <lysandre@huggingface.co>
Date: Thu, 29 Oct 2020 21:08:22 +0000
Subject: [PATCH 31/31] Update examples/language-modeling/README.md

---
 examples/language-modeling/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/language-modeling/README.md b/examples/language-modeling/README.md
index 6e6dc699ae09..9adf78db805f 100644
--- a/examples/language-modeling/README.md
+++ b/examples/language-modeling/README.md
@@ -63,7 +63,7 @@ python run_language_modeling.py \
     --whole_word_mask
 ```
 
-For Chinese models, it's same with English model with only `--mlm`. If using whole-word masking, we need to generate a reference files, cause it's char level.
+For Chinese models, it's same with English model with only `--mlm`. If using whole-word masking, we need to generate a reference files, because it's char level.
 
 **Q :** Why ref file ?