init
This commit is contained in:
l.gabrysiak
parent
c617cb3f95
commit
481cae2a61
100
finding.py
100
finding.py
|
|
@ -1,100 +0,0 @@
|
|||
import weaviate
|
||||
from weaviate.connect import ConnectionParams
|
||||
import re
|
||||
|
||||
# Konfiguracja klienta Weaviate
|
||||
client = weaviate.WeaviateClient(
|
||||
connection_params=ConnectionParams.from_params(
|
||||
http_host="weaviate",
|
||||
http_port=8080,
|
||||
http_secure=False,
|
||||
grpc_host="weaviate",
|
||||
grpc_port=50051,
|
||||
grpc_secure=False,
|
||||
)
|
||||
)
|
||||
client.connect()
|
||||
|
||||
# Pobierz kolekcję
|
||||
collection = client.collections.get("Document")
|
||||
|
||||
def extract_full_article(content, article_number):
|
||||
pattern = rf"Art\.\s*{article_number}\..*?(?=Art\.\s*\d+\.|\Z)"
|
||||
match = re.search(pattern, content, re.DOTALL)
|
||||
if match:
|
||||
return match.group(0).strip()
|
||||
return None
|
||||
|
||||
def extract_relevant_fragment(content, query, context_size=100):
|
||||
article_match = re.match(r"Art\.\s*(\d+)", query)
|
||||
if article_match:
|
||||
article_number = article_match.group(1)
|
||||
full_article = extract_full_article(content, article_number)
|
||||
if full_article:
|
||||
return full_article
|
||||
|
||||
index = content.lower().find(query.lower())
|
||||
if index != -1:
|
||||
start = max(0, index - context_size)
|
||||
end = min(len(content), index + len(query) + context_size)
|
||||
return f"...{content[start:end]}..."
|
||||
return content[:200] + "..."
|
||||
|
||||
def vector_search(query, limit=5):
|
||||
print(f"\nWyszukiwanie wektorowe dla zapytania: '{query}'")
|
||||
response = collection.query.near_text(
|
||||
query=query,
|
||||
limit=limit
|
||||
)
|
||||
for obj in response.objects:
|
||||
print(f"UUID: {obj.uuid}")
|
||||
relevant_fragment = extract_relevant_fragment(obj.properties['content'], query)
|
||||
print(f"Relewantny fragment:\n{relevant_fragment}")
|
||||
print(f"Nazwa pliku: {obj.properties['fileName']}")
|
||||
print("---")
|
||||
|
||||
def hybrid_search(query, limit=5, alpha=0.5):
|
||||
print(f"\nWyszukiwanie hybrydowe dla zapytania: '{query}'")
|
||||
response = collection.query.hybrid(
|
||||
query=query,
|
||||
alpha=alpha,
|
||||
limit=limit
|
||||
)
|
||||
for obj in response.objects:
|
||||
print(f"UUID: {obj.uuid}")
|
||||
relevant_fragment = extract_relevant_fragment(obj.properties['content'], query)
|
||||
print(f"Relewantny fragment:\n{relevant_fragment}")
|
||||
print(f"Nazwa pliku: {obj.properties['fileName']}")
|
||||
print("---")
|
||||
|
||||
|
||||
#exists = client.collections.exists("Document")
|
||||
#print(f"Czy kolekcja 'Document' istnieje: {exists}")
|
||||
|
||||
#schema = collection.config.get()
|
||||
#print(f"Nazwa kolekcji: {schema.name}")
|
||||
#print("Właściwości:")
|
||||
#for prop in schema.properties:
|
||||
# print(f"- {prop.name}: {prop.data_type}")
|
||||
|
||||
#collection = client.collections.get("Document")
|
||||
#count = collection.aggregate.over_all(total_count=True).total_count
|
||||
#print(f"Liczba obiektów w kolekcji: {count}")
|
||||
|
||||
#results = collection.query.fetch_objects(limit=5)
|
||||
#for obj in results.objects:
|
||||
# print(f"UUID: {obj.uuid}")
|
||||
# print(f"Nazwa pliku: {obj.properties['fileName']}")
|
||||
# print(f"Zawartość: {obj.properties['content'][:100]}...") # Pierwsze 100 znaków
|
||||
# print("---")
|
||||
|
||||
|
||||
# Przykładowe użycie
|
||||
queries = ["Art. 154", "urlop wypoczynkowy", "Państwowa Inspekcja Pracy"]
|
||||
|
||||
for query in queries:
|
||||
vector_search(query)
|
||||
hybrid_search(query)
|
||||
|
||||
# Zamknij połączenie
|
||||
client.close()
|
||||
157
manual.py
157
manual.py
|
|
@ -1,157 +0,0 @@
|
|||
import os
|
||||
import weaviate
|
||||
from weaviate.connect import ConnectionParams
|
||||
from weaviate.collections import Collection
|
||||
from weaviate.classes.config import Configure, Property, DataType
|
||||
from weaviate.collections.classes.filters import Filter
|
||||
import pytesseract
|
||||
from PIL import Image
|
||||
from docx import Document
|
||||
from pypdf import PdfReader
|
||||
import textract
|
||||
import hashlib
|
||||
|
||||
# Konfiguracja
|
||||
REPO_PATH = "/home/ably.do/docs"
|
||||
WEAVIATE_URL = "http://weaviate:8080"
|
||||
|
||||
client = weaviate.WeaviateClient(
|
||||
connection_params=ConnectionParams.from_params(
|
||||
http_host="weaviate",
|
||||
http_port=8080,
|
||||
http_secure=False,
|
||||
grpc_host="weaviate",
|
||||
grpc_port=50051,
|
||||
grpc_secure=False,
|
||||
)
|
||||
)
|
||||
|
||||
def read_text_file(file_path):
|
||||
with open(file_path, 'r', encoding='utf-8') as file:
|
||||
return file.read()
|
||||
|
||||
def read_docx(file_path):
|
||||
doc = Document(file_path)
|
||||
return ' '.join([paragraph.text for paragraph in doc.paragraphs])
|
||||
|
||||
def read_pdf(file_path):
|
||||
reader = PdfReader(file_path)
|
||||
return ' '.join([page.extract_text() for page in reader.pages])
|
||||
|
||||
def read_image(file_path):
|
||||
return pytesseract.image_to_string(Image.open(file_path))
|
||||
|
||||
def read_file(file_path):
|
||||
_, ext = os.path.splitext(file_path.lower())
|
||||
if ext in ['.txt', '.md']:
|
||||
return read_text_file(file_path)
|
||||
elif ext == '.docx':
|
||||
return read_docx(file_path)
|
||||
elif ext == '.pdf':
|
||||
return read_pdf(file_path)
|
||||
elif ext in ['.png', '.jpg', '.jpeg', '.gif', '.bmp']:
|
||||
return read_image(file_path)
|
||||
elif ext in ['.doc', '.rtf']:
|
||||
return textract.process(file_path).decode('utf-8')
|
||||
else:
|
||||
return None
|
||||
|
||||
def generate_content_hash(content):
|
||||
return hashlib.sha256(content.encode('utf-8')).hexdigest()
|
||||
|
||||
def add_to_weaviate(file_name, content, content_hash):
|
||||
try:
|
||||
collection = client.collections.get("Document")
|
||||
|
||||
# Poprawne użycie klasy Filter
|
||||
filters = Filter.by_property("fileName").equal(file_name)
|
||||
|
||||
# Sprawdzenie, czy dokument już istnieje
|
||||
existing_docs = collection.query.fetch_objects(filters=filters)
|
||||
|
||||
if existing_docs.objects:
|
||||
print(f"Dokument {file_name} już istnieje w bazie.")
|
||||
return
|
||||
|
||||
# Dodanie nowego dokumentu
|
||||
collection.data.insert(
|
||||
properties={
|
||||
"fileName": file_name,
|
||||
"content": content,
|
||||
"contentHash": content_hash
|
||||
}
|
||||
)
|
||||
print(f"Dodano dokument {file_name} do Weaviate.")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Błąd podczas dodawania {file_name} do Weaviate: {e}")
|
||||
|
||||
def process_file(file_path):
|
||||
if not os.path.exists(file_path):
|
||||
print(f"Plik nie istnieje: {file_path}")
|
||||
return
|
||||
|
||||
try:
|
||||
content = read_file(file_path)
|
||||
if content:
|
||||
file_name = os.path.basename(file_path)
|
||||
content_hash = generate_content_hash(content)
|
||||
add_to_weaviate(file_name, content, content_hash)
|
||||
else:
|
||||
print(f"Plik jest pusty lub nie można go odczytać: {file_path}")
|
||||
except Exception as e:
|
||||
print(f"Błąd podczas przetwarzania pliku {file_path}: {str(e)}")
|
||||
|
||||
def load_all_documents():
|
||||
print("Wczytywanie wszystkich dokumentów z katalogu...")
|
||||
for root, dirs, files in os.walk(REPO_PATH):
|
||||
for file in files:
|
||||
process_file(os.path.join(root, file))
|
||||
print("Zakończono wczytywanie dokumentów.")
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Upewnij się, że kolekcja "Document" istnieje w Weaviate
|
||||
client.connect()
|
||||
try:
|
||||
# Sprawdzenie, czy kolekcja istnieje i czy należy ją usunąć
|
||||
collection_name = "Document"
|
||||
if client.collections.exists(collection_name):
|
||||
print(f"Usuwanie istniejącej kolekcji '{collection_name}' (CLEAR_COLLECTION=true)...")
|
||||
client.collections.delete(collection_name)
|
||||
print(f"Kolekcja '{collection_name}' została usunięta.")
|
||||
else:
|
||||
print(f"Kolekcja '{collection_name}' nie istnieje.")
|
||||
|
||||
# Tworzenie kolekcji od nowa, jeśli została usunięta lub nie istniała
|
||||
if not client.collections.exists(collection_name):
|
||||
print(f"Tworzenie nowej kolekcji '{collection_name}'...")
|
||||
client.collections.create(
|
||||
name=collection_name,
|
||||
properties=[
|
||||
Property(name="content", data_type=DataType.TEXT),
|
||||
Property(name="fileName", data_type=DataType.TEXT),
|
||||
Property(name="contentHash", data_type=DataType.TEXT) # Nowe pole
|
||||
],
|
||||
vectorizer_config=Configure.Vectorizer.text2vec_transformers()
|
||||
)
|
||||
print(f"Kolekcja '{collection_name}' została utworzona.")
|
||||
|
||||
# Wczytanie dokumentów po utworzeniu nowej kolekcji
|
||||
print("Wczytywanie dokumentów do nowej kolekcji...")
|
||||
load_all_documents()
|
||||
print("Wszystkie dokumenty zostały wgrane.")
|
||||
|
||||
else:
|
||||
print("Kolekcja już istnieje. Pominięto jej ponowne tworzenie.")
|
||||
|
||||
# Sprawdzenie, czy kolekcja jest pusta i ewentualne wczytanie dokumentów
|
||||
collection = client.collections.get(collection_name)
|
||||
if collection.aggregate.over_all(total_count=True).total_count == 0:
|
||||
print("Kolekcja jest pusta. Wczytywanie dokumentów...")
|
||||
load_all_documents()
|
||||
print("Wszystkie dokumenty zostały wgrane do istniejącej kolekcji.")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Wystąpił błąd podczas operacji na kolekcji '{collection_name}': {e}")
|
||||
|
||||
client.close()
|
||||
Loading…
Reference in New Issue