POKASH.PL
/
ably.do
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

mod gemma

This commit is contained in:
l.gabrysiak 2025-02-26 13:25:17 +01:00
parent 310c882b1d
commit 1241d01180
1 changed files with 20 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
gemma.py
View File

@ -36,13 +36,18 @@ def create_training_data():
dataset = create_training_data() dataset = create_training_data()
# 5️⃣ Ładowanie modelu Gemma 2 7B # Podział danych na treningowe i ewaluacyjne
split_dataset = dataset.train_test_split(test_size=0.25)
train_dataset = split_dataset["train"]
eval_dataset = split_dataset["test"]
# 5️⃣ Ładowanie modelu Gemma 2B
device = "cuda" if torch.cuda.is_available() else "cpu" device = "cuda" if torch.cuda.is_available() else "cpu"
model_name = "google/gemma-2-2b" model_name = "google/gemma-2-2b"
model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16).to(device) model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16).to(device)
tokenizer = AutoTokenizer.from_pretrained(model_name) tokenizer = AutoTokenizer.from_pretrained(model_name)
# 6️⃣ Konfiguracja LoRA dla efektywnego treningu # 6️⃣ Konfiguracja LoRA
lora_config = LoraConfig( lora_config = LoraConfig(
r=8, lora_alpha=32, lora_dropout=0.1, bias="none", task_type="CAUSAL_LM" r=8, lora_alpha=32, lora_dropout=0.1, bias="none", task_type="CAUSAL_LM"
) )
@ -59,23 +64,24 @@ def tokenize_function(examples):
max_length=max_length max_length=max_length
) )
tokenized_dataset = dataset.map(tokenize_function, batched=True) tokenized_train = train_dataset.map(tokenize_function, batched=True)
tokenized_eval = eval_dataset.map(tokenize_function, batched=True)
# 8️⃣ Parametry treningu # 8️⃣ Parametry treningu
training_args = TrainingArguments( training_args = TrainingArguments(
output_dir="./results", output_dir="./results",
evaluation_strategy="steps", # Zmienione na "steps" eval_strategy="steps", # Ewaluacja co określoną liczbę kroków
eval_steps=500, # Dodane eval_steps=500, # Ewaluacja co 500 kroków
save_strategy="steps", # Zmienione na "steps" save_strategy="steps", # Zapis modelu co określoną liczbę kroków
save_steps=500, # Dodane, musi być takie samo jak eval_steps lub jego wielokrotność save_steps=500, # Zapis modelu co 500 kroków
learning_rate=2e-5, learning_rate=2e-5,
per_device_train_batch_size=2, per_device_train_batch_size=2,
per_device_eval_batch_size=2, per_device_eval_batch_size=2,
num_train_epochs=5, num_train_epochs=5,
weight_decay=0.01, weight_decay=0.01,
load_best_model_at_end=True, load_best_model_at_end=True, # Wczytaj najlepszy model na końcu
metric_for_best_model="loss", # lub inna metryka, którą chcesz optymalizować metric_for_best_model="loss", # Kryterium wyboru najlepszego modelu
greater_is_better=False, # Ustaw na True, jeśli wyższa wartość metryki jest lepsza greater_is_better=False, # Niższy loss = lepszy model
) )
# 9️⃣ Data Collator # 9️⃣ Data Collator
@ -88,14 +94,15 @@ data_collator = DataCollatorForLanguageModeling(
trainer = Trainer( trainer = Trainer(
model=model, model=model,
args=training_args, args=training_args,
train_dataset=tokenized_dataset, train_dataset=tokenized_train,
eval_dataset=tokenized_eval, # Dodany zestaw ewaluacyjny
data_collator=data_collator, data_collator=data_collator,
) )
trainer.train() trainer.train()
# 1️⃣1️⃣ Zapisanie dostrojonego modelu # 1️⃣1️⃣ Zapis modelu
model.save_pretrained("./trained_model/gemma") model.save_pretrained("./trained_model/gemma")
tokenizer.save_pretrained("./trained_model/gemma") tokenizer.save_pretrained("./trained_model/gemma")
print("✅ Model został wytrenowany i zapisany!") print("✅ Model został wytrenowany i zapisany!")