POKASH.PL
/
ably.do
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

mod

This commit is contained in:
l.gabrysiak 2025-02-25 15:02:36 +01:00
parent 999b9ade54
commit 204dd4421a
1 changed files with 17 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
hft.py
View File

@ -18,6 +18,11 @@ torch.cuda.empty_cache()
# Logowanie do Hugging Face Hub # Logowanie do Hugging Face Hub
login(token="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") login(token="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")
os.environ["TOKENIZERS_PARALLELISM"] = "false" os.environ["TOKENIZERS_PARALLELISM"] = "false"
os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"
def free_memory():
torch.cuda.empty_cache()
torch.cuda.ipc_collect()
class SourceMapper: class SourceMapper:
def __init__(self): def __init__(self):
@ -54,7 +59,7 @@ def extract_text_from_file(file_path):
with open(file_path, 'rb') as file: with open(file_path, 'rb') as file:
reader = PyPDF2.PdfReader(file) reader = PyPDF2.PdfReader(file)
for page in reader.pages: for page in reader.pages:
text += page.extract_text() text += page.extract_text() or ""
return text return text
elif ext in ['.doc', '.docx']: elif ext in ['.doc', '.docx']:
return docx2txt.process(file_path) return docx2txt.process(file_path)
@ -76,7 +81,7 @@ def prepare_dataset(directory, catalog_path, source_mapper):
doc_type = identify_legal_document(file, file_catalog) doc_type = identify_legal_document(file, file_catalog)
if doc_type != "Opracowanie własne": if doc_type != "Opracowanie własne":
articles = re.split(r'(Art\.?\s+\d+[\.\s])', text) articles = re.split(r'(Art\.\s+\d+\.)', text)
for i in range(1, len(articles), 2): for i in range(1, len(articles), 2):
article_number = articles[i].strip() article_number = articles[i].strip()
article_content = articles[i+1].strip() if i+1 < len(articles) else "" article_content = articles[i+1].strip() if i+1 < len(articles) else ""
@ -137,14 +142,6 @@ class CustomModel(GPTNeoForCausalLM):
outputs.logits += source_embeds outputs.logits += source_embeds
return outputs return outputs
class CustomTrainer(Trainer):
def compute_loss(self, model, inputs, return_outputs=False, **kwargs):
labels = inputs.pop("labels")
with autocast():
source_idx = inputs.pop("source_idx")
outputs = model(**inputs, labels=labels, source_idx=source_idx)
return (outputs.loss, outputs) if return_outputs else outputs.loss
source_mapper = SourceMapper() source_mapper = SourceMapper()
model_name = "EleutherAI/gpt-neo-2.7B" model_name = "EleutherAI/gpt-neo-2.7B"
tokenizer = AutoTokenizer.from_pretrained(model_name) tokenizer = AutoTokenizer.from_pretrained(model_name)
@ -152,7 +149,7 @@ tokenizer.pad_token = tokenizer.eos_token
data = prepare_dataset("files", "file_catalog.json", source_mapper) data = prepare_dataset("files", "file_catalog.json", source_mapper)
dataset = Dataset.from_list(data) dataset = Dataset.from_list(data)
tokenized_dataset = dataset.map(tokenize_function, batched=True, batch_size=32) tokenized_dataset = dataset.map(tokenize_function, batched=True, batch_size=16)
config = AutoModelForCausalLM.from_pretrained(model_name).config config = AutoModelForCausalLM.from_pretrained(model_name).config
model = CustomModel.from_pretrained(model_name) model = CustomModel.from_pretrained(model_name)
@ -164,26 +161,28 @@ model.gradient_checkpointing_enable()
training_args = TrainingArguments( training_args = TrainingArguments(
output_dir="./results", output_dir="./results",
num_train_epochs=3, num_train_epochs=3,
gradient_accumulation_steps=4, gradient_accumulation_steps=8,
learning_rate=2e-5, learning_rate=2e-5,
fp16=True, fp16=True,
logging_steps=100, logging_steps=50,
save_strategy="steps", save_strategy="steps",
save_steps=1000, save_steps=500,
report_to="none", per_device_train_batch_size=2,
per_device_train_batch_size=4, per_device_eval_batch_size=2,
per_device_eval_batch_size=4,
logging_dir='./logs' logging_dir='./logs'
) )
trainer = CustomTrainer( trainer = Trainer(
model=model, model=model,
args=training_args, args=training_args,
train_dataset=tokenized_dataset, train_dataset=tokenized_dataset,
data_collator=custom_collate_fn data_collator=custom_collate_fn
) )
trainer.train() trainer.train()
free_memory()
# Funkcja generująca odpowiedź # Funkcja generująca odpowiedź
def generate_answer(question, model, tokenizer, source_mapper, max_length=200): def generate_answer(question, model, tokenizer, source_mapper, max_length=200):
inputs = tokenizer(question, return_tensors="pt", truncation=True, max_length=512) inputs = tokenizer(question, return_tensors="pt", truncation=True, max_length=512)