POKASH.PL
/
ably.do
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

mod gpt

This commit is contained in:
l.gabrysiak 2025-02-26 10:35:35 +01:00
parent 16dc460786
commit 1243759c5a
1 changed files with 9 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
gpt.py
View File

@ -44,6 +44,7 @@ def prepare_dataset_from_file(file_path):
return formatted_articles return formatted_articles
def main(): def main():
# Inicjalizacja tokenizera # Inicjalizacja tokenizera
tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME) tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
@ -66,11 +67,11 @@ def main():
tokenized["labels"] = tokenized["input_ids"].clone() tokenized["labels"] = tokenized["input_ids"].clone()
return tokenized return tokenized
tokenized_dataset = dataset.map(tokenize_function, batched=True, remove_columns=dataset.column_names) tokenized_dataset = dataset.map(tokenize_function, batched=True)
# Model i data collator # Model i data collator
model = AutoModelForCausalLM.from_pretrained(MODEL_NAME) model = AutoModelForCausalLM.from_pretrained(MODEL_NAME)
model.resize_token_embeddings(len(tokenizer)) model.resize_token_embeddings(len(tokenizer), mean_resizing=False)
data_collator = DataCollatorForLanguageModeling( data_collator = DataCollatorForLanguageModeling(
tokenizer=tokenizer, tokenizer=tokenizer,
@ -80,17 +81,13 @@ def main():
# Konfiguracja treningu # Konfiguracja treningu
training_args = TrainingArguments( training_args = TrainingArguments(
output_dir="./results", output_dir="./results",
num_train_epochs=15, # Zwiększono liczbę epok num_train_epochs=16, # Zwiększono liczbę epok
per_device_train_batch_size=4, # Zwiększono rozmiar batcha per_device_train_batch_size=2,
learning_rate=2e-5, # Zmniejszono learning rate learning_rate=2e-5, #precyzja uczenia
weight_decay=0.01, # Dodano weight decay
logging_steps=10, logging_steps=10,
save_steps=500, # Dodano zapisywanie modelu co 500 kroków
eval_steps=500, # Dodano ewaluację co 500 kroków
evaluation_strategy="steps",
load_best_model_at_end=True, # Ładowanie najlepszego modelu na końcu
report_to="none", report_to="none",
save_total_limit=2, # Ograniczenie liczby zapisywanych checkpointów save_strategy="no",
load_best_model_at_end=True, # Ładowanie najlepszego modelu na końcu
) )
# Trainer # Trainer
@ -98,7 +95,6 @@ def main():
model=model, model=model,
args=training_args, args=training_args,
train_dataset=tokenized_dataset, train_dataset=tokenized_dataset,
eval_dataset=tokenized_dataset, # Używamy tego samego zbioru do ewaluacji
data_collator=data_collator data_collator=data_collator
) )
@ -108,4 +104,4 @@ def main():
tokenizer.save_pretrained("./trained_model/gpt") tokenizer.save_pretrained("./trained_model/gpt")
if __name__ == "__main__": if __name__ == "__main__":
main() main()