POKASH.PL
/
ably.do
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

mod

This commit is contained in:
l.gabrysiak 2025-02-26 00:12:11 +01:00
parent 4957a2898b
commit 4ac8c40417
1 changed files with 8 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
hft.py
View File

@ -17,7 +17,6 @@ os.environ['TORCH_USE_CUDA_DSA'] = '1'
os.environ["TOKENIZERS_PARALLELISM"] = "false" os.environ["TOKENIZERS_PARALLELISM"] = "false"
login(token="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") login(token="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")
# Nowe tokeny specjalne
CITATION_START = "▌▌CITATION_START" CITATION_START = "▌▌CITATION_START"
CITATION_END = "▌▌CITATION_END" CITATION_END = "▌▌CITATION_END"
@ -180,7 +179,6 @@ class CustomModel(nn.Module):
self.base_model = AutoModelForCausalLM.from_pretrained(model_name, config=config) self.base_model = AutoModelForCausalLM.from_pretrained(model_name, config=config)
self.source_embedding = nn.Embedding(10000, config.hidden_size, padding_idx=-1) self.source_embedding = nn.Embedding(10000, config.hidden_size, padding_idx=-1)
# Dodaj specjalne tokeny i zaktualizuj embeddings
tokenizer.add_special_tokens({'additional_special_tokens': [CITATION_START, CITATION_END]}) tokenizer.add_special_tokens({'additional_special_tokens': [CITATION_START, CITATION_END]})
self.base_model.resize_token_embeddings(len(tokenizer)) self.base_model.resize_token_embeddings(len(tokenizer))
@ -215,7 +213,7 @@ class CustomDataCollator(DataCollatorForLanguageModeling):
batch = super().torch_call(examples) batch = super().torch_call(examples)
if "source_idx" in examples[0]: if "source_idx" in examples[0]:
source_idx = torch.tensor([ex["source_idx"] for ex in examples]) source_idx = torch.stack([ex["source_idx"] for ex in examples])
batch["source_idx"] = source_idx batch["source_idx"] = source_idx
return batch return batch
@ -224,11 +222,9 @@ def main():
source_mapper = SourceMapper() source_mapper = SourceMapper()
model_name = "crumb/nano-mistral" model_name = "crumb/nano-mistral"
# Inicjalizacja tokenizera
tokenizer = AutoTokenizer.from_pretrained(model_name) tokenizer = AutoTokenizer.from_pretrained(model_name)
tokenizer.pad_token = tokenizer.eos_token tokenizer.pad_token = tokenizer.eos_token
# Przygotowanie danych
catalog_path = "catalog.json" catalog_path = "catalog.json"
data = prepare_dataset("docs", catalog_path, source_mapper) data = prepare_dataset("docs", catalog_path, source_mapper)
@ -246,16 +242,18 @@ def main():
max_length=512, max_length=512,
return_tensors="pt" return_tensors="pt"
) )
source_idx = torch.tensor(examples["source_idx"], dtype=torch.long)
return { return {
"input_ids": tokenized["input_ids"].squeeze(), "input_ids": tokenized["input_ids"].squeeze(),
"attention_mask": tokenized["attention_mask"].squeeze(), "attention_mask": tokenized["attention_mask"].squeeze(),
"labels": tokenized["input_ids"].squeeze().clone(), "labels": tokenized["input_ids"].squeeze().clone(),
"source_idx": examples["source_idx"] "source_idx": source_idx
} }
tokenized_dataset = dataset.map(tokenize_function, batched=True, batch_size=16) tokenized_dataset = dataset.map(tokenize_function, batched=True, batch_size=16)
# Inicjalizacja modelu z tokenizerem
model = CustomModel(model_name, tokenizer) model = CustomModel(model_name, tokenizer)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu") device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model.to(device) model.to(device)