POKASH.PL
/
ably.do
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

mod allegro

This commit is contained in:
l.gabrysiak 2025-02-28 21:43:06 +01:00
parent 029662e9d1
commit 447de65d83
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
allegro.py
View File

@ -11,7 +11,7 @@ dataset = load_dataset("wmt16", "ro-en")
# Przetwórz dane do formatu odpowiedniego dla modelu # Przetwórz dane do formatu odpowiedniego dla modelu
def tokenize_function(examples): def tokenize_function(examples):
return tokenizer(examples['translation'], truncation=True, padding='max_length', max_length=128) return tokenizer(examples['translation']['ro'], examples['translation']['en'], truncation=True, padding='max_length', max_length=128)
tokenized_datasets = dataset.map(tokenize_function, batched=True) tokenized_datasets = dataset.map(tokenize_function, batched=True)