POKASH.PL
/
ably.do
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

mod allegro

This commit is contained in:
l.gabrysiak 2025-02-28 20:45:55 +01:00
parent 2980d74be4
commit 124e904c31
1 changed files with 7 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
allegro.py
View File

@ -8,32 +8,24 @@ from datasets import Dataset
from peft import LoraConfig, get_peft_model from peft import LoraConfig, get_peft_model
from transformers import AutoModelForSeq2SeqLM, AutoTokenizer, TrainingArguments, Trainer, DataCollatorForSeq2Seq from transformers import AutoModelForSeq2SeqLM, AutoTokenizer, TrainingArguments, Trainer, DataCollatorForSeq2Seq
import weaviate import weaviate
from weaviate.client import WeaviateClient
from weaviate.connect import ConnectionParams
# 1️⃣ Inicjalizacja modelu do embeddingów # 1️⃣ Inicjalizacja modelu do embeddingów
embed_model = SentenceTransformer("all-MiniLM-L6-v2") embed_model = SentenceTransformer("all-MiniLM-L6-v2")
# 2️⃣ Połączenie z Weaviate i pobranie dokumentów # 2️⃣ Połączenie z Weaviate i pobranie dokumentów
client = WeaviateClient( client = weaviate.Client(
connection_params=ConnectionParams.from_params( url="http://weaviate:8080" # Dostosuj URL do swojego środowiska
http_host="weaviate",
http_port=8080,
http_secure=False,
grpc_host="weaviate",
grpc_port=50051,
grpc_secure=False,
)
) )
collection_name = "Document" # Zakładam, że to jest nazwa Twojej kolekcji collection_name = "Document" # Zakładam, że to jest nazwa Twojej kolekcji
result = ( response = (
client.query.get(collection_name, ["content"]) client.query
.get(collection_name, ["content"])
.with_additional(["id"]) .with_additional(["id"])
.do() .do()
) )
documents = [item['content'] for item in result['data']['Get'][collection_name]] documents = [item['content'] for item in response['data']['Get'][collection_name]]
# 3️⃣ Generowanie embeddingów # 3️⃣ Generowanie embeddingów
embeddings = embed_model.encode(documents) embeddings = embed_model.encode(documents)
@ -82,7 +74,7 @@ tokenized_eval = eval_dataset.map(tokenize_function, batched=True)
# 8️⃣ Parametry treningu # 8️⃣ Parametry treningu
training_args = TrainingArguments( training_args = TrainingArguments(
output_dir="./results", output_dir="./results",
eval_strategy="steps", evaluation_strategy="steps",
eval_steps=500, eval_steps=500,
save_strategy="steps", save_strategy="steps",
save_steps=500, save_steps=500,