mod

hft.py

@@ -1,8 +1,6 @@
 import os
 import torch
 import torch.nn as nn
-from transformers import AutoTokenizer, AutoModelForCausalLM, TrainingArguments, Trainer, DataCollatorForLanguageModeling
-from datasets import Dataset
 import re
 import json
 import PyPDF2
@@ -10,252 +8,286 @@ import docx2txt
 import pytesseract
 from PIL import Image
 from collections import defaultdict
+from transformers import (
+    AutoTokenizer,
+    AutoModelForCausalLM,
+    TrainingArguments,
+    DataCollatorForLanguageModeling
+)
+from datasets import Dataset
 from huggingface_hub import login
 
 # Konfiguracja
-os.environ['TORCH_USE_CUDA_DSA'] = '1'
 os.environ["TOKENIZERS_PARALLELISM"] = "false"
-login(token="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")
+login(token="TWÓJ_TOKEN_HUGGINGFACE")
 
-class SourceMapper:
+class LegalAITrainer:
     def __init__(self):
-        self.source_to_idx = defaultdict(lambda: len(self.source_to_idx))
+        self.source_mapper = defaultdict(lambda: len(self.source_mapper))
         self.idx_to_source = {}
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 
-    def add_source(self, source):
+    class SourceMapper:
-        if source and source not in self.source_to_idx:
+        def __init__(self):
-            idx = self.source_to_idx[source]
+            self.source_to_idx = defaultdict(lambda: len(self.source_to_idx))
-            self.idx_to_source[idx] = source
+            self.idx_to_source = {}
 
-    def get_idx(self, source):
+        def add_source(self, source):
-        return self.source_to_idx[source] if source else -1
+            if source and source not in self.source_to_idx:
+                idx = self.source_to_idx[source]
+                self.idx_to_source[idx] = source
 
-    def get_source(self, idx):
+        def get_idx(self, source):
-        return self.idx_to_source.get(idx, "Unknown")
+            return self.source_to_idx[source] if source else -1
 
-def load_file_catalog(catalog_path):
+        def get_source(self, idx):
-    try:
+            return self.idx_to_source.get(idx, "Unknown")
-        with open(catalog_path, 'r', encoding='utf-8') as file:
-            return json.load(file)
-    except Exception as e:
-        print(f"Błąd wczytywania katalogu plików: {str(e)}")
-        return {}
 
-def identify_legal_document(filename, file_catalog):
+    class LegalModel(nn.Module):
-    base_name = os.path.splitext(filename)[0].lower()
+        def __init__(self, model_name, config):
-    return file_catalog.get(base_name, "Opracowanie własne")
+            super().__init__()
+            self.base_model = AutoModelForCausalLM.from_pretrained(model_name, config=config)
+            self.source_embedding = nn.Embedding(100000, config.hidden_size, padding_idx=-1)
+            self.confidence_layer = nn.Linear(config.hidden_size, 1)
             
-def extract_text_from_file(file_path):
+            # Freeze base model
-    try:
+            for param in self.base_model.parameters():
-        _, ext = os.path.splitext(file_path)
+                param.requires_grad = False
-        ext = ext.lower()
                 
-        if ext in ['.txt', '.md']:
+            # Trainable components
-            with open(file_path, 'r', encoding='utf-8') as file:
+            for layer in [self.source_embedding, self.confidence_layer]:
-                return file.read()
+                for param in layer.parameters():
-        elif ext == '.pdf':
+                    param.requires_grad = True
-            text = ""
+
-            try:
+        def forward(self, input_ids=None, attention_mask=None, labels=None, source_idx=None):
-                with open(file_path, 'rb') as file:
+            if source_idx is not None:
-                    reader = PyPDF2.PdfReader(file)
+                source_idx = torch.clamp(source_idx, 0, self.source_embedding.num_embeddings-1)
+                source_embeds = self.source_embedding(source_idx).unsqueeze(1)
+                inputs_embeds = self.base_model.get_input_embeddings()(input_ids) + source_embeds
+                outputs = self.base_model(
+                    inputs_embeds=inputs_embeds,
+                    attention_mask=attention_mask,
+                    labels=labels
+                )
+            else:
+                outputs = self.base_model(
+                    input_ids=input_ids,
+                    attention_mask=attention_mask,
+                    labels=labels
+                )
+                
+            confidence = torch.sigmoid(self.confidence_layer(outputs.hidden_states[-1].mean(dim=1)))
+            return {
+                "loss": outputs.loss,
+                "logits": outputs.logits,
+                "confidence": confidence,
+                "hidden_states": outputs.hidden_states
+            }
+
+    def load_file_catalog(self, catalog_path):
+        try:
+            with open(catalog_path, 'r', encoding='utf-8') as f:
+                return json.load(f)
+        except Exception as e:
+            print(f"Błąd ładowania katalogu: {str(e)}")
+            return {}
+
+    def extract_text(self, file_path):
+        ext = os.path.splitext(file_path)[1].lower()
+        try:
+            if ext in ['.txt', '.md']:
+                with open(file_path, 'r', encoding='utf-8') as f:
+                    return f.read()
+            elif ext == '.pdf':
+                text = ""
+                with open(file_path, 'rb') as f:
+                    reader = PyPDF2.PdfReader(f)
                     for page in reader.pages:
                         text += page.extract_text() or ""
-            except Exception as e:
+                return text
-                print(f"Błąd PDF: {str(e)}")
+            elif ext in ['.doc', '.docx']:
-            return text
+                return docx2txt.process(file_path)
-        elif ext in ['.doc', '.docx']:
+            elif ext in ['.jpg', '.jpeg', '.png']:
-            return docx2txt.process(file_path)
+                return pytesseract.image_to_string(Image.open(file_path))
-        elif ext in ['.jpg', '.jpeg', '.png', '.bmp', '.tiff']:
+            else:
-            return pytesseract.image_to_string(Image.open(file_path))
+                return ""
-        else:
+        except Exception as e:
-            print(f"Nieobsługiwany format pliku: {ext}")
+            print(f"Błąd przetwarzania {file_path}: {str(e)}")
             return ""
-    except Exception as e:
-        print(f"Błąd ekstrakcji tekstu: {str(e)}")
-        return ""
 
-def prepare_dataset(directory, catalog_path, source_mapper):
+    def prepare_data(self, data_dir, catalog_path):
-    file_catalog = load_file_catalog(catalog_path)
+        catalog = self.load_file_catalog(catalog_path)
-    data = []
+        data = []
+        source_mapper = self.SourceMapper()
 
-    print(f"\n{'='*50}\nDIAGNOSTYKA DANYCH\n{'='*50}")
+        for root, _, files in os.walk(data_dir):
+            for file in files:
+                file_path = os.path.join(root, file)
+                text = self.extract_text(file_path)
                 
-    for root, _, files in os.walk(directory):
+                if not text:
-        for file in files:
-            file_path = os.path.join(root, file)
-            print(f"\nPrzetwarzanie pliku: {file_path}")
-            
-            try:
-                text = extract_text_from_file(file_path)
-                if not text.strip():
-                    print("Pominięto - brak tekstu")
                     continue
                 
-                print(f"Długość tekstu: {len(text)} znaków")
+                doc_type = catalog.get(os.path.splitext(file)[0].lower(), "Opracowanie własne")
-                
-                doc_type = identify_legal_document(file, file_catalog)
-                print(f"Rozpoznany typ dokumentu: {doc_type}")
                 
                 if doc_type != "Opracowanie własne":
-                    articles = re.split(r'(?i)(Art[\.\s]+\d+[\.\s]?)', text)
+                    articles = re.split(r'(?i)(Art\.\s*\d+[a-z]*)', text)
-                    articles = [a.strip() for a in articles if a.strip()]
+                    for i in range(1, len(articles), 2):
+                        art_num = articles[i].strip()
+                        content = articles[i+1].strip()
                         
-                    print(f"Znaleziono {len(articles)} fragmentów")
+                        if len(content) < 100:
-                    
-                    for i in range(0, len(articles)-1, 2):
-                        article_number = articles[i]
-                        article_content = articles[i+1]
-                        
-                        if len(article_content) < 50:
                             continue
                             
-                        source = f"{doc_type}, {article_number}"
+                        source = f"{doc_type}, {art_num}"
                         source_mapper.add_source(source)
                         data.append({
-                            "text": f"{article_number} {article_content}",
+                            "text": f"[LEGAL] {art_num} {content}",
-                            "source_idx": source_mapper.get_idx(source)
+                            "source_idx": source_mapper.get_idx(source),
+                            "is_legal": 1
                         })
                 else:
-                    clean_text = re.sub(r'\s+', ' ', text).strip()
+                    chunks = [f"[GENERAL] {text[i:i+512]}" for i in range(0, len(text), 512)]
-                    chunks = [clean_text[i:i+512] for i in range(0, len(clean_text), 512)]
-                    chunks = [c for c in chunks if c.strip()]
-                    
                     for chunk in chunks:
                         data.append({
                             "text": chunk,
-                            "source_idx": -1
+                            "source_idx": -1,
+                            "is_legal": 0
                         })
-                    print(f"Dodano {len(chunks)} chunków")
         
-            except Exception as e:
+        return Dataset.from_dict({k: [d[k] for d in data] for k in data[0]}), source_mapper
-                print(f"Błąd podczas przetwarzania pliku: {str(e)}")
-                continue
 
-    print(f"\nPodsumowanie przygotowania danych:")
+    def train(self, model_name="crumb/nano-mistral", data_dir="data", catalog_path="catalog.json"):
-    print(f"Łączna liczba przykładów: {len(data)}")
+        # Przygotowanie danych
-    if data:
+        dataset, source_mapper = self.prepare_data(data_dir, catalog_path)
-        print("Przykładowy wpis:")
+        tokenizer = AutoTokenizer.from_pretrained(model_name)
-        print(json.dumps(data[0], indent=2, ensure_ascii=False))
+        tokenizer.pad_token = tokenizer.eos_token
-    else:
-        print("BRAK DANYCH - sprawdź diagnostykę powyżej")
 
-    return data
+        # Tokenizacja
-
+        def tokenize_fn(examples):
-class CustomModel(nn.Module):
+            tokenized = tokenizer(
-    def __init__(self, model_name, config):
+                examples["text"],
-        super().__init__()
+                padding="max_length",
-        self.base_model = AutoModelForCausalLM.from_pretrained(model_name, config=config)
+                truncation=True,
-        self.source_embedding = nn.Embedding(10000, config.hidden_size, padding_idx=-1)
+                max_length=512,
-        
+                return_tensors="pt"
-        for param in self.base_model.parameters():
-            param.requires_grad = False
-        for param in self.base_model.get_output_embeddings().parameters():
-            param.requires_grad = True
-            
-    def forward(self, input_ids=None, attention_mask=None, labels=None, source_idx=None, **kwargs):
-        if source_idx is not None:
-            valid_indices = torch.clamp(source_idx, 0, self.source_embedding.num_embeddings-1)
-            source_embeds = self.source_embedding(valid_indices).unsqueeze(1)
-            inputs_embeds = self.base_model.get_input_embeddings()(input_ids) + source_embeds
-            return self.base_model(
-                inputs_embeds=inputs_embeds,
-                attention_mask=attention_mask,
-                labels=labels,
-                **kwargs
             )
-        return self.base_model(
+            return {
-            input_ids=input_ids,
+                "input_ids": tokenized["input_ids"].squeeze(),
-            attention_mask=attention_mask,
+                "attention_mask": tokenized["attention_mask"].squeeze(),
-            labels=labels,
+                "labels": tokenized["input_ids"].squeeze().clone(),
-            **kwargs
+                "source_idx": examples["source_idx"]
+            }
+
+        tokenized_dataset = dataset.map(tokenize_fn, batched=True, batch_size=16)
+
+        # Inicjalizacja modelu
+        config = AutoModelForCausalLM.from_pretrained(model_name).config
+        model = self.LegalModel(model_name, config).to(self.device)
+
+        # Konfiguracja treningu
+        training_args = TrainingArguments(
+            output_dir="./legal_ai_model",
+            num_train_epochs=5,
+            per_device_train_batch_size=2,
+            gradient_accumulation_steps=4,
+            learning_rate=2e-5,
+            fp16=torch.cuda.is_available(),
+            logging_steps=50,
+            save_strategy="steps",
+            save_steps=500,
+            report_to="none",
+            remove_unused_columns=False
         )
 
-    def generate(self, *args, **kwargs):
+        # Customowy Trainer
-        return self.base_model.generate(*args, **kwargs)
+        class LegalTrainer(Trainer):
+            def compute_loss(self, model, inputs, return_outputs=False):
+                outputs = model(**inputs)
+                loss = outputs.loss
                 
-class CustomDataCollator(DataCollatorForLanguageModeling):
+                # Confidence loss
-    def torch_call(self, examples):
+                target_conf = (inputs["source_idx"] != -1).float()
-        # Przetwórz podstawowe pola
+                conf_loss = nn.BCELoss()(outputs.confidence.squeeze(), target_conf)
-        input_ids = torch.stack([torch.tensor(ex["input_ids"]) for ex in examples])
-        attention_mask = torch.stack([torch.tensor(ex["attention_mask"]) for ex in examples])
-        labels = torch.stack([torch.tensor(ex["labels"]) for ex in examples])
                 
-        batch = {
+                total_loss = loss + 0.7*conf_loss
-            "input_ids": input_ids,
+                return (total_loss, outputs) if return_outputs else total_loss
-            "attention_mask": attention_mask,
-            "labels": labels
-        }
 
-        # Dodaj source_idx jeśli istnieje
+        # Trening
-        if "source_idx" in examples[0]:
+        trainer = LegalTrainer(
-            source_idx = torch.stack([torch.tensor(ex["source_idx"]) for ex in examples])
+            model=model,
-            batch["source_idx"] = source_idx
+            args=training_args,
+            train_dataset=tokenized_dataset,
+            data_collator=DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)
+        )
 
-        return batch
+        print("Rozpoczęcie treningu...")
+        trainer.train()
         
-def main():
+        # Zapisz model
-    source_mapper = SourceMapper()
+        model.save_pretrained("./trained_legal_ai")
-    model_name = "crumb/nano-mistral"
+        tokenizer.save_pretrained("./trained_legal_ai")
-    tokenizer = AutoTokenizer.from_pretrained(model_name)
+        with open("./trained_legal_ai/source_mapper.json", "w") as f:
-    tokenizer.pad_token = tokenizer.eos_token
+            json.dump(source_mapper.idx_to_source, f)
 
-    # Przygotowanie danych
+        print("Trening zakończony i model zapisany!")
-    catalog_path = "file_catalog.json"
-    data = prepare_dataset("files", catalog_path, source_mapper)
 
-    if not data:
+    def generate_response(self, prompt, confidence_threshold=0.65):
-        print("\nBrak danych do treningu!")
+        # Ładowanie modelu
-        return
+        model = self.LegalModel.from_pretrained("./trained_legal_ai", 
+                                              config=AutoModelForCausalLM.from_pretrained("crumb/nano-mistral").config)
+        tokenizer = AutoTokenizer.from_pretrained("./trained_legal_ai")
+        model.to(self.device)
         
-    #dataset = Dataset.from_list(data)
+        # Ładowanie mapowania źródeł
-    dataset = Dataset.from_dict({k: [d[k] for d in data] for k in data[0]})
+        with open("./trained_legal_ai/source_mapper.json", "r") as f:
+            source_mapper = json.load(f)
 
-
+        # Przygotowanie wejścia
-    def tokenize_function(examples):
+        inputs = tokenizer(
-        tokenized = tokenizer(
+            f"[PROMPT] {prompt} [RESPONSE]",
-            examples["text"],
+            return_tensors="pt",
-            truncation=True,
-            padding="max_length",
             max_length=512,
-            return_tensors="pt"
+            truncation=True
-        )
+        ).to(self.device)
-        return {
-            "input_ids": tokenized["input_ids"].squeeze(),
-            "attention_mask": tokenized["attention_mask"].squeeze(),
-            "labels": tokenized["input_ids"].squeeze().clone(),
-            "source_idx": examples["source_idx"]  # Dodano bez konwersji do tensora
-        }
 
-    tokenized_dataset = dataset.map(tokenize_function, batched=True, batch_size=16)
+        # Generacja
+        with torch.no_grad():
+            outputs = model.generate(
+                input_ids=inputs.input_ids,
+                attention_mask=inputs.attention_mask,
+                max_length=512,
+                do_sample=True,
+                temperature=0.7,
+                top_k=50,
+                pad_token_id=tokenizer.eos_token_id,
+                output_scores=True,
+                return_dict_in_generate=True
+            )
 
-    model = CustomModel(model_name, AutoModelForCausalLM.from_pretrained(model_name).config)
+        # Analiza wyników
-    model.source_mapper = source_mapper
+        full_text = tokenizer.decode(outputs.sequences[0], skip_special_tokens=True)
-    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        confidence = torch.sigmoid(outputs.scores[-1][:, tokenizer.eos_token_id]).item()
-    model.to(device)
         
-    training_args = TrainingArguments(
+        # Ekstrakcja i weryfikacja źródeł
-        output_dir="./results",
+        citations = list(set(re.findall(r"Art\.\s*\d+[a-z]*", full_text)))
-        num_train_epochs=3,
+        verified = [c for c in citations if any(c in s for s in source_mapper.values())]
-        per_device_train_batch_size=2,
-        gradient_accumulation_steps=4,
-        learning_rate=2e-5,
-        fp16=torch.cuda.is_available(),
-        logging_steps=10,
-        save_strategy="steps",
-        save_steps=1000,
-        report_to="none",
-        remove_unused_columns=False
-    )
 
-    trainer = Trainer(
+        if confidence < confidence_threshold or not verified:
-        model=model,
+            return "Nie mogę udzielić jednoznacznej odpowiedzi na podstawie dostępnych danych."
-        args=training_args,
+        else:
-        train_dataset=tokenized_dataset,
+            return f"{full_text}\n\nPotwierdzone źródła: {', '.join(verified)}"
-        data_collator=CustomDataCollator(tokenizer=tokenizer, mlm=False)
-    )
-
-    print("\nRozpoczęcie treningu...")
-    trainer.train()
 
 if __name__ == "__main__":
-    main()
+    legal_ai = LegalAITrainer()
+    
+    # Etap 1: Trening
+    legal_ai.train(
+        model_name="crumb/nano-mistral",
+        data_dir="./legal_docs",
+        catalog_path="./catalog.json"
+    )
+    
+    # Etap 2: Testowanie
+    test_prompt = "Ile dni urlopu przysługuje po 5 latach pracy w pełnym wymiarze?"
+    print(legal_ai.generate_response(test_prompt))