POKASH.PL
/
ably.do
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

trener mod

This commit is contained in:
l.gabrysiak 2025-02-25 13:17:17 +01:00
parent a380f06555
commit 329d76d072
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hft.py
View File

@ -13,6 +13,7 @@ import json
from huggingface_hub import login from huggingface_hub import login
login(f"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") login(f"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")
os.environ["TOKENIZERS_PARALLELISM"] = "false"
def load_file_catalog(catalog_path): def load_file_catalog(catalog_path):
with open(catalog_path, 'r', encoding='utf-8') as file: with open(catalog_path, 'r', encoding='utf-8') as file:
@ -101,8 +102,7 @@ class CustomTrainer(Trainer):
def compute_loss(self, model, inputs, return_outputs=False): def compute_loss(self, model, inputs, return_outputs=False):
labels = inputs.pop("labels") labels = inputs.pop("labels")
source = inputs.pop("source") source = inputs.pop("source")
source_ids = torch.tensor([hash(s) % 1000 for s in source], device=model.device) outputs = model(**inputs, labels=labels)
outputs = model(**inputs, labels=labels, source=source_ids)
loss = outputs.loss loss = outputs.loss
return (loss, outputs) if return_outputs else loss return (loss, outputs) if return_outputs else loss