From 0df49895cf5b8188d917dca99b3ef4bdbcfab3fe Mon Sep 17 00:00:00 2001 From: "l.gabrysiak" Date: Wed, 26 Feb 2025 00:28:34 +0100 Subject: [PATCH] mod gpt --- gpt.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gpt.py b/gpt.py index ed66541..19f7358 100644 --- a/gpt.py +++ b/gpt.py @@ -16,7 +16,10 @@ def prepare_simple_dataset(): def main(): # Inicjalizacja tokenizera - tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME) + tokenizer = AutoTokenizer.from_pretrained( + MODEL_NAME, + mean_resizing=False + ) tokenizer.add_special_tokens({"additional_special_tokens": SPECIAL_TOKENS}) tokenizer.pad_token = tokenizer.eos_token