POKASH.PL
/
ably.do
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

mod

This commit is contained in:
l.gabrysiak 2025-02-25 17:11:05 +01:00
parent 7c24c381e0
commit ce550ad79d
1 changed files with 10 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
hft.py
View File

@ -15,7 +15,6 @@ from huggingface_hub import login
login(token="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") login(token="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")
os.environ["TOKENIZERS_PARALLELISM"] = "false" os.environ["TOKENIZERS_PARALLELISM"] = "false"
# Nowa klasa do zarządzania źródłami
class SourceMapper: class SourceMapper:
def __init__(self): def __init__(self):
self.source_to_idx = defaultdict(lambda: len(self.source_to_idx)) self.source_to_idx = defaultdict(lambda: len(self.source_to_idx))
@ -119,7 +118,7 @@ class CustomModel(nn.Module):
super().__init__() super().__init__()
self.base_model = AutoModelForCausalLM.from_pretrained(model_name, config=config) self.base_model = AutoModelForCausalLM.from_pretrained(model_name, config=config)
self.source_embedding = nn.Embedding( self.source_embedding = nn.Embedding(
num_embeddings=1000, # Maksymalna liczba unikalnych źródeł num_embeddings=1000,
embedding_dim=config.hidden_size, embedding_dim=config.hidden_size,
padding_idx=-1 padding_idx=-1
) )
@ -133,8 +132,9 @@ class CustomModel(nn.Module):
) )
if source_idx is not None: if source_idx is not None:
# Dodaj embedding źródła do logits print("outputs.logits shape:", outputs.logits.shape)
source_embeds = self.source_embedding(source_idx).unsqueeze(1) source_embeds = self.source_embedding(source_idx).unsqueeze(1).expand(-1, outputs.logits.size(1), -1)
print("source_embeds shape:", source_embeds.shape)
outputs.logits += source_embeds outputs.logits += source_embeds
return outputs return outputs
@ -149,7 +149,7 @@ class CustomTrainer(Trainer):
# Inicjalizacja komponentów # Inicjalizacja komponentów
source_mapper = SourceMapper() source_mapper = SourceMapper()
model_name = "crumb/nano-mistral" #"google/gemma-2-2b" model_name = "crumb/nano-mistral"
tokenizer = AutoTokenizer.from_pretrained(model_name) tokenizer = AutoTokenizer.from_pretrained(model_name)
tokenizer.pad_token = tokenizer.eos_token tokenizer.pad_token = tokenizer.eos_token
@ -162,6 +162,7 @@ tokenized_dataset = dataset.map(tokenize_function, batched=True, batch_size=8)
# Inicjalizacja modelu # Inicjalizacja modelu
config = AutoModelForCausalLM.from_pretrained(model_name).config config = AutoModelForCausalLM.from_pretrained(model_name).config
model = CustomModel(model_name, config) model = CustomModel(model_name, config)
model.to("cpu")
# Konfiguracja treningu # Konfiguracja treningu
training_args = TrainingArguments( training_args = TrainingArguments(
@ -171,11 +172,10 @@ training_args = TrainingArguments(
gradient_accumulation_steps=4, gradient_accumulation_steps=4,
learning_rate=2e-5, learning_rate=2e-5,
fp16=True, fp16=True,
logging_steps=1, # Częstsze logowanie logging_steps=1,
logging_dir="./logs", # Katalog na logi logging_dir="./logs",
save_strategy="steps", save_strategy="steps",
save_steps=1000, save_steps=1000,
#report_to="none"
) )
# Trening # Trening
@ -183,7 +183,7 @@ trainer = CustomTrainer(
model=model, model=model,
args=training_args, args=training_args,
train_dataset=tokenized_dataset, train_dataset=tokenized_dataset,
data_collator=custom_collate_fn, # Użyj niestandardowego collate_fn data_collator=custom_collate_fn,
) )
trainer.train() trainer.train()
@ -211,4 +211,4 @@ def generate_answer(question, model, tokenizer, source_mapper, max_length=200):
# Przykład użycia # Przykład użycia
question = "Ile dni urlopu przysługuje pracownikowi?" question = "Ile dni urlopu przysługuje pracownikowi?"
answer = generate_answer(question, model, tokenizer, source_mapper) answer = generate_answer(question, model, tokenizer, source_mapper)
print(answer) print(answer)