diff --git a/gpt.py b/gpt.py index ed66541..19f7358 100644 --- a/gpt.py +++ b/gpt.py @@ -16,7 +16,10 @@ def prepare_simple_dataset(): def main(): # Inicjalizacja tokenizera - tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME) + tokenizer = AutoTokenizer.from_pretrained( + MODEL_NAME, + mean_resizing=False + ) tokenizer.add_special_tokens({"additional_special_tokens": SPECIAL_TOKENS}) tokenizer.pad_token = tokenizer.eos_token