POKASH.PL
/
ably.do
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

mod gpt

This commit is contained in:
l.gabrysiak 2025-02-26 00:28:34 +01:00
parent 2dd8198c3a
commit 0df49895cf
1 changed files with 4 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
gpt.py
View File

@ -16,7 +16,10 @@ def prepare_simple_dataset():
def main():
# Inicjalizacja tokenizera
tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
tokenizer = AutoTokenizer.from_pretrained(
MODEL_NAME,
mean_resizing=False
)
tokenizer.add_special_tokens({"additional_special_tokens": SPECIAL_TOKENS})
tokenizer.pad_token = tokenizer.eos_token