From cd784436efc6215d97829bdecd24fe598e3f6755 Mon Sep 17 00:00:00 2001
From: davidmezzetti <561939+davidmezzetti@users.noreply.github.com>
Date: Tue, 5 Aug 2025 16:16:36 -0400
Subject: [PATCH] Track indexing progress in benchmark script

---
 examples/benchmarks.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/examples/benchmarks.py b/examples/benchmarks.py
index 506efd17..fc48bb2f 100644
--- a/examples/benchmarks.py
+++ b/examples/benchmarks.py
@@ -23,6 +23,7 @@ from elasticsearch import Elasticsearch
 from elasticsearch.helpers import bulk
 from pytrec_eval import RelevanceEvaluator
 from rank_bm25 import BM25Okapi
+from tqdm.auto import tqdm
 
 from txtai.embeddings import Embeddings
 from txtai.pipeline import LLM, RAG, Tokenizer
@@ -111,8 +112,16 @@ class Index:
         Iterates over the dataset yielding a row at a time for indexing.
         """
 
-        with open(f"{self.path}/corpus.jsonl", encoding="utf-8") as f:
-            for line in f:
+        # Data file
+        path = f"{self.path}/corpus.jsonl"
+
+        # Get total count
+        with open(path, encoding="utf-8") as f:
+            total = sum(1 for _ in f)
+
+        # Yield data
+        with open(path, encoding="utf-8") as f:
+            for line in tqdm(f, total=total):
                 row = json.loads(line)
                 text = f'{row["title"]}. {row["text"]}' if row.get("title") else row["text"]
                 if text: