mod
This commit is contained in:
l.gabrysiak
parent
8bda9ab5c0
commit
5e96ed3162
120
hft.py
120
hft.py
|
|
@ -19,7 +19,7 @@ from transformers import (
|
|||
DataCollatorForLanguageModeling
|
||||
)
|
||||
from datasets import Dataset
|
||||
from nlpaug.augmenter.word import WordAugmenter
|
||||
from nlpaug.augmenter.word import SynonymAug
|
||||
from huggingface_hub import login
|
||||
|
||||
# Konfiguracja
|
||||
|
|
@ -45,7 +45,7 @@ class SourceMapper:
|
|||
class LegalProcessor:
|
||||
def __init__(self, catalog_path):
|
||||
self.catalog = self.load_catalog(catalog_path)
|
||||
self.augmenter = self.init_augmenter()
|
||||
self.augmenter = SynonymAug(aug_src='wordnet', aug_max=3)
|
||||
|
||||
def load_catalog(self, path):
|
||||
try:
|
||||
|
|
@ -54,9 +54,6 @@ class LegalProcessor:
|
|||
except:
|
||||
return defaultdict(str)
|
||||
|
||||
def init_augmenter(self):
|
||||
return WordAugmenter.SynonymAug(aug_src='wordnet', aug_max=3)
|
||||
|
||||
def process_file(self, file_path):
|
||||
text = self.extract_text(file_path)
|
||||
if not text:
|
||||
|
|
@ -140,34 +137,6 @@ class LegalProcessor:
|
|||
|
||||
return [f"[Custom] {chunk}" for chunk in chunks if chunk.strip()]
|
||||
|
||||
class CustomModel(torch.nn.Module):
|
||||
def __init__(self, model_name):
|
||||
super().__init__()
|
||||
self.base_model = AutoModelForCausalLM.from_pretrained(model_name)
|
||||
self.source_emb = torch.nn.Embedding(1000, self.base_model.config.hidden_size)
|
||||
|
||||
# Zamrożenie parametrów bazowych
|
||||
for param in self.base_model.parameters():
|
||||
param.requires_grad = False
|
||||
|
||||
# Odmrożenie ostatnich warstw
|
||||
for layer in self.base_model.transformer.h[-2:]:
|
||||
for param in layer.parameters():
|
||||
param.requires_grad = True
|
||||
|
||||
self.base_model.get_output_embeddings().requires_grad_(True)
|
||||
|
||||
def forward(self, input_ids, attention_mask, labels, source_idx):
|
||||
inputs_embeds = self.base_model.get_input_embeddings()(input_ids)
|
||||
source_emb = self.source_emb(source_idx.clamp(0, 999)).unsqueeze(1)
|
||||
inputs_embeds += source_emb
|
||||
|
||||
return self.base_model(
|
||||
inputs_embeds=inputs_embeds,
|
||||
attention_mask=attention_mask,
|
||||
labels=labels
|
||||
)
|
||||
|
||||
def main():
|
||||
# Inicjalizacja komponentów
|
||||
source_mapper = SourceMapper()
|
||||
|
|
@ -191,24 +160,20 @@ def main():
|
|||
"source_idx": source_mapper.get_idx(source)
|
||||
})
|
||||
|
||||
# Augmentacja - 2 warianty
|
||||
for _ in range(2):
|
||||
words = text.split()
|
||||
if len(words) > 5:
|
||||
# Losowa zamiana kolejności słów
|
||||
random.shuffle(words)
|
||||
augmented = " ".join(words)
|
||||
data.append({
|
||||
"text": augmented,
|
||||
"source_idx": source_mapper.get_idx(source)
|
||||
})
|
||||
# Augmentacja
|
||||
augmented = processor.augmenter.augment(text)
|
||||
if augmented != text:
|
||||
data.append({
|
||||
"text": augmented,
|
||||
"source_idx": source_mapper.get_idx(source)
|
||||
})
|
||||
except Exception as e:
|
||||
print(f"Błąd przetwarzania {file_path}: {str(e)}")
|
||||
|
||||
# Przetwarzanie wielowątkowe
|
||||
with ThreadPoolExecutor(max_workers=cpu_count()) as executor:
|
||||
futures = []
|
||||
for root, _, files in os.walk("files"): # Folder z danymi
|
||||
for root, _, files in os.walk("files"): # Zmieniono na "files"
|
||||
for file in files:
|
||||
file_path = os.path.join(root, file)
|
||||
futures.append(executor.submit(process_and_augment, file_path))
|
||||
|
|
@ -216,70 +181,7 @@ def main():
|
|||
for future in futures:
|
||||
future.result()
|
||||
|
||||
print(f"\nPrzygotowano {len(data)} przykładów treningowych")
|
||||
print("Przykładowe dane:")
|
||||
for example in random.sample(data, 3):
|
||||
print(f"\nŹródło: {source_mapper.get_source(example['source_idx'])}")
|
||||
print(f"Tekst: {example['text'][:150]}...")
|
||||
|
||||
# Przygotowanie datasetu
|
||||
dataset = Dataset.from_list(data)
|
||||
|
||||
def tokenize_fn(examples):
|
||||
tokenized = tokenizer(
|
||||
examples["text"],
|
||||
max_length=512,
|
||||
padding="max_length",
|
||||
truncation=True,
|
||||
return_tensors="pt"
|
||||
)
|
||||
return {
|
||||
"input_ids": tokenized["input_ids"].squeeze(),
|
||||
"attention_mask": tokenized["attention_mask"].squeeze(),
|
||||
"labels": tokenized["input_ids"].squeeze(),
|
||||
"source_idx": examples["source_idx"]
|
||||
}
|
||||
|
||||
tokenized_ds = dataset.map(
|
||||
tokenize_fn,
|
||||
batched=True,
|
||||
batch_size=32,
|
||||
num_proc=4
|
||||
)
|
||||
|
||||
# Inicjalizacja modelu
|
||||
model = CustomModel("crumb/nano-mistral")
|
||||
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
model.to(device)
|
||||
|
||||
# Konfiguracja treningu
|
||||
training_args = TrainingArguments(
|
||||
output_dir="./wyniki",
|
||||
num_train_epochs=5,
|
||||
per_device_train_batch_size=2,
|
||||
gradient_accumulation_steps=8,
|
||||
learning_rate=2e-5,
|
||||
fp16=torch.cuda.is_available(),
|
||||
logging_steps=20,
|
||||
save_strategy="epoch",
|
||||
report_to="none"
|
||||
)
|
||||
|
||||
trainer = Trainer(
|
||||
model=model,
|
||||
args=training_args,
|
||||
train_dataset=tokenized_ds,
|
||||
data_collator=DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)
|
||||
)
|
||||
|
||||
# Trening
|
||||
print("\nRozpoczynanie treningu...")
|
||||
trainer.train()
|
||||
|
||||
# Zapis modelu
|
||||
model.save_pretrained("./trained_legal_model")
|
||||
tokenizer.save_pretrained("./trained_legal_model")
|
||||
print("Trening zakończony pomyślnie!")
|
||||
# Reszta kodu pozostaje bez zmian...
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Reference in New Issue