POKASH.PL
/
ably.do
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

mod

This commit is contained in:
l.gabrysiak 2025-02-25 14:29:02 +01:00
parent 21634e53cc
commit c0d5772742
1 changed files with 7 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
hft.py
View File

@ -1,7 +1,8 @@
import os import os
import torch import torch
import torch.nn as nn import torch.nn as nn
from transformers import AutoTokenizer, AutoModelForCausalLM, TrainingArguments, Trainer #from transformers import AutoTokenizer, AutoModelForCausalLM, TrainingArguments, Trainer
from transformers import GPTNeoForCausalLM # Zmiana importu
from datasets import Dataset from datasets import Dataset
from PIL import Image from PIL import Image
import re import re
@ -118,11 +119,11 @@ def custom_collate_fn(batch):
return {"input_ids": input_ids, "attention_mask": attention_mask, "labels": labels, "source_idx": source_idx} return {"input_ids": input_ids, "attention_mask": attention_mask, "labels": labels, "source_idx": source_idx}
class CustomModel(AutoModelForCausalLM): class CustomModel(GPTNeoForCausalLM): # Zmiana klasy bazowej
def __init__(self, config): def __init__(self, config):
super().__init__(config) super().__init__(config)
self.source_embedding = nn.Embedding( self.source_embedding = nn.Embedding(
num_embeddings=1000, # Maksymalna liczba unikalnych źródeł num_embeddings=1000,
embedding_dim=config.hidden_size, embedding_dim=config.hidden_size,
padding_idx=-1 padding_idx=-1
) )
@ -136,7 +137,6 @@ class CustomModel(AutoModelForCausalLM):
) )
if source_idx is not None: if source_idx is not None:
# Dodajemy embedding źródła do hidden states
source_embeds = self.source_embedding(source_idx).unsqueeze(1) source_embeds = self.source_embedding(source_idx).unsqueeze(1)
outputs.logits += source_embeds outputs.logits += source_embeds
@ -163,7 +163,9 @@ tokenized_dataset = dataset.map(tokenize_function, batched=True, batch_size=32)
# Inicjalizacja modelu # Inicjalizacja modelu
config = AutoModelForCausalLM.from_pretrained(model_name).config config = AutoModelForCausalLM.from_pretrained(model_name).config
model = CustomModel.from_pretrained(model_name, config=config) #model = CustomModel.from_pretrained(model_name, config=config)
model = CustomModel.from_pretrained(model_name)
model.resize_token_embeddings(len(tokenizer))
model.gradient_checkpointing_enable() model.gradient_checkpointing_enable()
# Konfiguracja treningu # Konfiguracja treningu