a couple of explore scripts for: adding two special tokens (<idiom>, </idiom>)

explore/explore_bart_for_conditional_generation.py

@@ -1,10 +0,0 @@
-
-from transformers import BartTokenizer, BartForConditionalGeneration
-
-
-def main():
-    pass
-
-
-if __name__ == '__main__':
-    main()

explore/explore_bart_tokenizer_add_special_tokens.py

@@ -0,0 +1,27 @@
+from transformers import BartTokenizer, BartForConditionalGeneration
+
+
+def main():
+    tokenizer = BartTokenizer.from_pretrained("facebook/bart-base")
+    bart = BartForConditionalGeneration.from_pretrained("facebook/bart-base")
+    num_added_tokens = tokenizer.add_special_tokens({
+        "additional_special_tokens": ["<idiom>", "</idiom>"],  # beginning and end of an idiom
+    })
+    print(num_added_tokens)
+    print(tokenizer.additional_special_tokens)  # more special tokens are added here
+    # and then you should resize the embedding table of your model
+    print(bart.model.shared.weight.shape)  # before
+    bart.resize_token_embeddings(len(tokenizer))
+    print(bart.model.shared.weight.shape)  # after
+
+
+if __name__ == '__main__':
+    main()
+
+"""
+2
+['<idiom>', '</idiom>']
+torch.Size([50265, 768])
+torch.Size([50267, 768])  # you can see that 2 more embedding vectors have been added here.
+later, you may want to save the tokenizer after you add the idiom special tokens.
+"""

explore/explore_bart_tokenizer_special_tokens.py

@@ -0,0 +1,36 @@
+from transformers import BartTokenizer
+
+
+def main():
+    tokenizer = BartTokenizer.from_pretrained("facebook/bart-base")
+    print(tokenizer.bos_token)
+    print(tokenizer.cls_token)
+    print(tokenizer.eos_token)
+    print(tokenizer.sep_token)
+    print(tokenizer.mask_token)
+    print(tokenizer.pad_token)
+    print(tokenizer.unk_token)
+
+
+"""
+<s>
+<s>
+</s>
+</s>
+<mask>
+<pad>
+<unk>
+
+right, so this is just like the symbols for BERT but in lowercase. 
+bos = cls 
+sep = eos
+would it be okay to use <idiom> = <sep>? 
+no, sep implies that a sentence somehow ends. 
+"""
+
+
+
+
+
+if __name__ == '__main__':
+    main()