From 2edb4eda95919ca3fb2bf319e908392a9207e575 Mon Sep 17 00:00:00 2001 From: "l.gabrysiak" Date: Tue, 25 Feb 2025 12:19:36 +0100 Subject: [PATCH] poprawa prepare_dataset --- hft.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hft.py b/hft.py index 4b4bca0..984f23a 100644 --- a/hft.py +++ b/hft.py @@ -109,8 +109,7 @@ tokenizer = AutoTokenizer.from_pretrained(model_name) model = CustomModel.from_pretrained(model_name) # Przygotowanie datasetu -catalog_path = "file_catalog.json" -data = prepare_dataset("files") +data = prepare_dataset("files", "file_catalog.json") dataset = load_dataset("dict", data=data) tokenized_dataset = dataset.map(tokenize_function, batched=True, remove_columns=dataset["train"].column_names)