POKASH.PL
/
ably.do
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

mod allegro

This commit is contained in:
l.gabrysiak 2025-02-28 21:44:24 +01:00
parent 447de65d83
commit 33eff363bc
1 changed files with 4 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
allegro.py
View File

@ -11,7 +11,10 @@ dataset = load_dataset("wmt16", "ro-en")
# Przetwórz dane do formatu odpowiedniego dla modelu # Przetwórz dane do formatu odpowiedniego dla modelu
def tokenize_function(examples): def tokenize_function(examples):
return tokenizer(examples['translation']['ro'], examples['translation']['en'], truncation=True, padding='max_length', max_length=128) # Jeśli 'translation' to lista słowników, np. [{'en': 'text1', 'ro': 'text1_translated'}, ...]
return tokenizer([example['en'] for example in examples['translation']],
[example['ro'] for example in examples['translation']],
truncation=True, padding='max_length', max_length=128)
tokenized_datasets = dataset.map(tokenize_function, batched=True) tokenized_datasets = dataset.map(tokenize_function, batched=True)