Not able to generate nDCG or MRR Metrics

[amitbcp]

from rankify.dataset.dataset import Document, Question, Answer, Context
from rankify.retrievers.retriever import Retriever
from rankify.models.reranking import Reranking
from rankify.metrics.metrics import Metrics


# Sample Documents
documents = [
    Document(question=Question("the cast of a good day to die hard?"), answers=Answer([
            "Jai Courtney",
            "Sebastian Koch",
            "Radivoje Bukvić",
            "Yuliya Snigir",
            "Sergei Kolesnikov",
            "Mary Elizabeth Winstead",
            "Bruce Willis"
        ]), contexts=[]),
    Document(question=Question("Who wrote Hamlet?"), answers=Answer(["Shakespeare"]), contexts=[])
]

# BM25 retrieval on Wikipedia
bm25_retriever_wiki = Retriever(method="bm25", n_docs=5, index_type="wiki")

retrieved_documents = bm25_retriever_wiki.retrieve(documents)

# Initialize the reranker
reranker = Reranking(method='upr', model_name='t5-base')
# Apply reranking
reranked_documents = reranker.rank(retrieved_documents)

metrics = Metrics(reranked_documents)

# Calculate retrieval metrics before reranking
before_ranking_metrics = metrics.calculate_retrieval_metrics(ks=[1, 5, 10, 20, 50, 100], use_reordered=False)
print(f"Before Ranking : {before_ranking_metrics}")

# Calculate retrieval metrics after reranking
after_ranking_metrics = metrics.calculate_retrieval_metrics(ks=[1, 5, 10, 20, 50, 100], use_reordered=True)
print(f"After Ranking : {after_ranking_metrics}")

# Calculate retrieval metrics after reranking
after_ranking_metrics_ndcg = metrics.calculate_trec_metrics(ndcg_cuts=[1, 5, 10, 20, 50, 100], use_reordered=True)
print(after_ranking_metrics)

Logs

Oct 16, 2025 6:26:59 PM org.apache.lucene.store.MemorySegmentIndexInputProvider <init>
INFO: Using MemorySegmentIndexInput with Java 21; to disable start with -Dorg.apache.lucene.store.MMapDirectory.enableMemorySegments=false

Retrieving 2 documents with BM25...
Batch search: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1[/1](http://144.25.31.124:8473/1) [00:00<00:00, 17.46it[/s](http://144.25.31.124:8473/s)]
Processing documents: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2[/2](http://144.25.31.124:8473/2) [00:00<00:00, 273.21it[/s](http://144.25.31.124:8473/s)]


Reranking Documents: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2[/2](http://144.25.31.124:8473/2) [00:00<00:00, 25.33it[/s](http://144.25.31.124:8473/s)]


Before Ranking : {'top_1': 100.0, 'top_5': 100.0, 'top_10': 100.0, 'top_20': 100.0, 'top_50': 100.0, 'top_100': 100.0}
After Ranking : {'top_1': 100.0, 'top_5': 100.0, 'top_10': 100.0, 'top_20': 100.0, 'top_50': 100.0, 'top_100': 100.0}


Error running `trec_eval`: Traceback (most recent call last):
  File "<frozen runpy>", line 198, in _run_module_as_main
  File "<frozen runpy>", line 88, in _run_code
  File "/home/aamita/miniconda3/envs/rerank_rankify/lib/python3.11/site-packages/pyserini/eval/trec_eval.py", line 79, in <module>
    args[-2] = get_qrels_file(args[-2])
               ^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/aamita/miniconda3/envs/rerank_rankify/lib/python3.11/site-packages/pyserini/search/_base.py", line 717, in get_qrels_file
    target_path = JRelevanceJudgments.getQrelsPath(JPath.of(qrels.path)).toString()
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "jnius/jnius_export_class.pxi", line 896, in jnius.JavaMethod.__call__
  File "jnius/jnius_export_class.pxi", line 1062, in jnius.JavaMethod.call_staticmethod
  File "jnius/jnius_utils.pxi", line 79, in jnius.check_exception
jnius.JavaException: JVM exception occurred: java.io.IOException: Error downloading topics from https://raw.githubusercontent.com/castorini/anserini-tools/master/topics-and-qrels/qrels.dl19-passage.txt
java.io.IOException: Error downloading topics from https://raw.githubusercontent.com/castorini/anserini-tools/master/topics-and-qrels/qrels.dl19-passage.txt
	io.anserini.eval.RelevanceJudgments.downloadQrels(RelevanceJudgments.java:203)
	io.anserini.eval.RelevanceJudgments.getQrelsPath(RelevanceJudgments.java:178)

Error running `trec_eval`: Traceback (most recent call last):
  File "<frozen runpy>", line 198, in _run_module_as_main
  File "<frozen runpy>", line 88, in _run_code
  File "/home/aamita/miniconda3/envs/rerank_rankify/lib/python3.11/site-packages/pyserini/eval/trec_eval.py", line 79, in <module>
    args[-2] = get_qrels_file(args[-2])
               ^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/aamita/miniconda3/envs/rerank_rankify/lib/python3.11/site-packages/pyserini/search/_base.py", line 717, in get_qrels_file
    target_path = JRelevanceJudgments.getQrelsPath(JPath.of(qrels.path)).toString()
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "jnius/jnius_export_class.pxi", line 896, in jnius.JavaMethod.__call__
  File "jnius/jnius_export_class.pxi", line 1062, in jnius.JavaMethod.call_staticmethod
  File "jnius/jnius_utils.pxi", line 79, in jnius.check_exception
jnius.JavaException: JVM exception occurred: java.io.IOException: Error downloading topics from https://raw.githubusercontent.com/castorini/anserini-tools/master/topics-and-qrels/qrels.dl19-passage.txt
java.io.IOException: Error downloading topics from https://raw.githubusercontent.com/castorini/anserini-tools/master/topics-and-qrels/qrels.dl19-passage.txt
	io.anserini.eval.RelevanceJudgments.downloadQrels(RelevanceJudgments.java:203)
	io.anserini.eval.RelevanceJudgments.getQrelsPath(RelevanceJudgments.java:178)

Error running `trec_eval`: Traceback (most recent call last):
  File "<frozen runpy>", line 198, in _run_module_as_main
  File "<frozen runpy>", line 88, in _run_code
  File "/home/aamita/miniconda3/envs/rerank_rankify/lib/python3.11/site-packages/pyserini/eval/trec_eval.py", line 79, in <module>
    args[-2] = get_qrels_file(args[-2])
               ^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/aamita/miniconda3/envs/rerank_rankify/lib/python3.11/site-packages/pyserini/search/_base.py", line 717, in get_qrels_file
    target_path = JRelevanceJudgments.getQrelsPath(JPath.of(qrels.path)).toString()
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "jnius/jnius_export_class.pxi", line 896, in jnius.JavaMethod.__call__
  File "jnius/jnius_export_class.pxi", line 1062, in jnius.JavaMethod.call_staticmethod
  File "jnius/jnius_utils.pxi", line 79, in jnius.check_exception
jnius.JavaException: JVM exception occurred: java.io.IOException: Error downloading topics from https://raw.githubusercontent.com/castorini/anserini-tools/master/topics-and-qrels/qrels.dl19-passage.txt
java.io.IOException: Error downloading topics from https://raw.githubusercontent.com/castorini/anserini-tools/master/topics-and-qrels/qrels.dl19-passage.txt
	io.anserini.eval.RelevanceJudgments.downloadQrels(RelevanceJudgments.java:203)
	io.anserini.eval.RelevanceJudgments.getQrelsPath(RelevanceJudgments.java:178)

Error running `trec_eval`: Traceback (most recent call last):
  File "<frozen runpy>", line 198, in _run_module_as_main
  File "<frozen runpy>", line 88, in _run_code
  File "/home/aamita/miniconda3/envs/rerank_rankify/lib/python3.11/site-packages/pyserini/eval/trec_eval.py", line 79, in <module>
    args[-2] = get_qrels_file(args[-2])
               ^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/aamita/miniconda3/envs/rerank_rankify/lib/python3.11/site-packages/pyserini/search/_base.py", line 717, in get_qrels_file
    target_path = JRelevanceJudgments.getQrelsPath(JPath.of(qrels.path)).toString()
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "jnius/jnius_export_class.pxi", line 896, in jnius.JavaMethod.__call__
  File "jnius/jnius_export_class.pxi", line 1062, in jnius.JavaMethod.call_staticmethod
  File "jnius/jnius_utils.pxi", line 79, in jnius.check_exception
jnius.JavaException: JVM exception occurred: java.io.IOException: Error downloading topics from https://raw.githubusercontent.com/castorini/anserini-tools/master/topics-and-qrels/qrels.dl19-passage.txt
java.io.IOException: Error downloading topics from https://raw.githubusercontent.com/castorini/anserini-tools/master/topics-and-qrels/qrels.dl19-passage.txt
	io.anserini.eval.RelevanceJudgments.downloadQrels(RelevanceJudgments.java:203)
	io.anserini.eval.RelevanceJudgments.getQrelsPath(RelevanceJudgments.java:178)

Error running `trec_eval`: Traceback (most recent call last):
  File "<frozen runpy>", line 198, in _run_module_as_main
  File "<frozen runpy>", line 88, in _run_code
  File "/home/aamita/miniconda3/envs/rerank_rankify/lib/python3.11/site-packages/pyserini/eval/trec_eval.py", line 79, in <module>
    args[-2] = get_qrels_file(args[-2])
               ^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/aamita/miniconda3/envs/rerank_rankify/lib/python3.11/site-packages/pyserini/search/_base.py", line 717, in get_qrels_file
    target_path = JRelevanceJudgments.getQrelsPath(JPath.of(qrels.path)).toString()
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "jnius/jnius_export_class.pxi", line 896, in jnius.JavaMethod.__call__
  File "jnius/jnius_export_class.pxi", line 1062, in jnius.JavaMethod.call_staticmethod
  File "jnius/jnius_utils.pxi", line 79, in jnius.check_exception
jnius.JavaException: JVM exception occurred: java.io.IOException: Error downloading topics from https://raw.githubusercontent.com/castorini/anserini-tools/master/topics-and-qrels/qrels.dl19-passage.txt
java.io.IOException: Error downloading topics from https://raw.githubusercontent.com/castorini/anserini-tools/master/topics-and-qrels/qrels.dl19-passage.txt
	io.anserini.eval.RelevanceJudgments.downloadQrels(RelevanceJudgments.java:203)
	io.anserini.eval.RelevanceJudgments.getQrelsPath(RelevanceJudgments.java:178)

Error running `trec_eval`: Traceback (most recent call last):
  File "<frozen runpy>", line 198, in _run_module_as_main
  File "<frozen runpy>", line 88, in _run_code
  File "/home/aamita/miniconda3/envs/rerank_rankify/lib/python3.11/site-packages/pyserini/eval/trec_eval.py", line 79, in <module>
    args[-2] = get_qrels_file(args[-2])
               ^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/aamita/miniconda3/envs/rerank_rankify/lib/python3.11/site-packages/pyserini/search/_base.py", line 717, in get_qrels_file
    target_path = JRelevanceJudgments.getQrelsPath(JPath.of(qrels.path)).toString()
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "jnius/jnius_export_class.pxi", line 896, in jnius.JavaMethod.__call__
  File "jnius/jnius_export_class.pxi", line 1062, in jnius.JavaMethod.call_staticmethod
  File "jnius/jnius_utils.pxi", line 79, in jnius.check_exception
jnius.JavaException: JVM exception occurred: java.io.IOException: Error downloading topics from https://raw.githubusercontent.com/castorini/anserini-tools/master/topics-and-qrels/qrels.dl19-passage.txt
java.io.IOException: Error downloading topics from https://raw.githubusercontent.com/castorini/anserini-tools/master/topics-and-qrels/qrels.dl19-passage.txt
	io.anserini.eval.RelevanceJudgments.downloadQrels(RelevanceJudgments.java:203)
	io.anserini.eval.RelevanceJudgments.getQrelsPath(RelevanceJudgments.java:178)

Error running `trec_eval`: Traceback (most recent call last):
  File "<frozen runpy>", line 198, in _run_module_as_main
  File "<frozen runpy>", line 88, in _run_code
  File "/home/aamita/miniconda3/envs/rerank_rankify/lib/python3.11/site-packages/pyserini/eval/trec_eval.py", line 79, in <module>
    args[-2] = get_qrels_file(args[-2])
               ^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/aamita/miniconda3/envs/rerank_rankify/lib/python3.11/site-packages/pyserini/search/_base.py", line 717, in get_qrels_file
    target_path = JRelevanceJudgments.getQrelsPath(JPath.of(qrels.path)).toString()
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "jnius/jnius_export_class.pxi", line 896, in jnius.JavaMethod.__call__
  File "jnius/jnius_export_class.pxi", line 1062, in jnius.JavaMethod.call_staticmethod
  File "jnius/jnius_utils.pxi", line 79, in jnius.check_exception
jnius.JavaException: JVM exception occurred: java.io.IOException: Error downloading topics from https://raw.githubusercontent.com/castorini/anserini-tools/master/topics-and-qrels/qrels.dl19-passage.txt
java.io.IOException: Error downloading topics from https://raw.githubusercontent.com/castorini/anserini-tools/master/topics-and-qrels/qrels.dl19-passage.txt
	io.anserini.eval.RelevanceJudgments.downloadQrels(RelevanceJudgments.java:203)
	io.anserini.eval.RelevanceJudgments.getQrelsPath(RelevanceJudgments.java:178)

Error running `trec_eval`: Traceback (most recent call last):
  File "<frozen runpy>", line 198, in _run_module_as_main
  File "<frozen runpy>", line 88, in _run_code
  File "/home/aamita/miniconda3/envs/rerank_rankify/lib/python3.11/site-packages/pyserini/eval/trec_eval.py", line 79, in <module>
    args[-2] = get_qrels_file(args[-2])
               ^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/aamita/miniconda3/envs/rerank_rankify/lib/python3.11/site-packages/pyserini/search/_base.py", line 717, in get_qrels_file
    target_path = JRelevanceJudgments.getQrelsPath(JPath.of(qrels.path)).toString()
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "jnius/jnius_export_class.pxi", line 896, in jnius.JavaMethod.__call__
  File "jnius/jnius_export_class.pxi", line 1062, in jnius.JavaMethod.call_staticmethod
  File "jnius/jnius_utils.pxi", line 79, in jnius.check_exception
jnius.JavaException: JVM exception occurred: java.io.IOException: Error downloading topics from https://raw.githubusercontent.com/castorini/anserini-tools/master/topics-and-qrels/qrels.dl19-passage.txt
java.io.IOException: Error downloading topics from https://raw.githubusercontent.com/castorini/anserini-tools/master/topics-and-qrels/qrels.dl19-passage.txt
	io.anserini.eval.RelevanceJudgments.downloadQrels(RelevanceJudgments.java:203)
	io.anserini.eval.RelevanceJudgments.getQrelsPath(RelevanceJudgments.java:178)

{'ndcg@1': 0.0, 'ndcg@5': 0.0, 'ndcg@10': 0.0, 'ndcg@20': 0.0, 'ndcg@50': 0.0, 'ndcg@100': 0.0, 'map@100': 0.0, 'mrr@10': 0.0}

[abdoelsayed2016]

The error occurs because calculate_trec_metrics() requires qrels to compute metrics like nDCG, MAP, and MRR. When using custom datasets (like the sample documents in the example), qrels are not available.

i will consider this in the next version by raise warming to user

def calculate_trec_metrics(self, ndcg_cuts=[1, 5, 10], use_reordered=True):

    if not self._has_qrels():
        print("Warning: TREC metrics require qrels. Skipping calculation.")
        return {f'ndcg@{k}': 0.0 for k in ndcg_cuts} | {'map@100': 0.0, 'mrr@10': 0.0}

[abdoelsayed2016]

please dont close the issue.

[amitbcp]

@abdoelsayed2016 - but we should be enabling users to compute the metrics for custom datasets and benchmarks as well right , otherwise it gets to restrictive ?

[abdoelsayed2016]

@amitbcp yes users can pass qrel file i will provide example how to use for custom datasets

[amitbcp]

Thanks that would be helpful

[amitbcp]

Also another thing to share, it might be worth taking a look at MTEB repo as they have retrieval and reranking metric completely in python -

1. https://github.com/embeddings-benchmark/mteb/blob/main/mteb/evaluation/evaluators/RetrievalEvaluator.py
2. https://github.com/embeddings-benchmark/mteb/blob/main/mteb/evaluation/evaluators/RerankingEvaluator.py
3. https://github.com/cvangysel/pytrec_eval

This would further eliminate java dependencies from pyserini if required

[abdoelsayed2016]

ok, sure I will look at them. Thanks very much

[amitbcp]

The BGE group also uses pytrec_eval for metrics -
https://github.com/FlagOpen/FlagEmbedding/tree/master/examples/evaluation

[amitbcp]

Hey @abdoelsayed2016 - I tried using a prebuilt index and that also has the same error -

from rankify.dataset.dataset import Document, Question, Answer, Context
from rankify.retrievers.retriever import Retriever
from rankify.models.reranking import Reranking
from rankify.metrics.metrics import Metrics

dataset = Dataset(retriever="bm25", dataset_name="nq-dev", n_docs=100)
documents = dataset.download(force_download=False)

# Initialize the reranker
reranker = Reranking(method='upr', model_name='t5-base')
# Apply reranking
reranked_documents = reranker.rank(documents)

metrics = Metrics(reranked_documents)

# Calculate retrieval metrics after reranking
after_ranking_metrics_ndcg = metrics.calculate_trec_metrics(ndcg_cuts=[1, 5, 10, 20, 50, 100], use_reordered=True)
print(after_ranking_metrics)

The above creates the same error which was created for custom dataset

[abdoelsayed2016]

@amitbcp It is the same as we discussed before, these datasets are not designed for these metrics, it is open qa, so you need to calculate the TOP metrics

if you want to use these metrics you can run the following code

import os
import json
import time
from rankify.dataset.dataset import Dataset
from rankify.models.reranking import Reranking
from rankify.metrics.metrics import Metrics

# Set CUDA environment variable
#os.environ["CUDA_VISIBLE_DEVICES"] = "0"

# Define methods and models
methods = {
    'upr': ["t5-small", ], #"t5-base", "t5-large", "t0-3b", "gpt2", "gpt2-medium", "gpt2-large"
    # 'rankgpt': ['Llama-3.2-1B', 'Llama-3.2-3B', 'Qwen2.5-7B', 'llamav3.1-8b', 'Mistral-7B-Instruct-v0.2', "Mistral-7B-Instruct-v0.3"],
    # 'flashrank': ["ms-marco-TinyBERT-L-2-v2", "ms-marco-MiniLM-L-12-v2", "ms-marco-MultiBERT-L-12", "rank-T5-flan", "ce-esci-MiniLM-L12-v2"],
    # 'monot5': ["monot5-base-msmarco", "monot5-base-msmarco-10k", "monot5-large-msmarco" , "monot5-large-msmarco-10k", "monot5-base-med-msmarco", "monot5-3b-med-msmarco",  "monot5-3b-msmarco-10k", "mt5-base-en-msmarco", "ptt5-base-pt-msmarco-10k-v2", "ptt5-base-pt-msmarco-100k-v2", "ptt5-base-en-pt-msmarco-100k-v2", "mt5-base-en-pt-msmarco-v2", "mt5-base-mmarco-v2", "mt5-base-en-pt-msmarco-v1", "mt5-base-mmarco-v1", "ptt5-base-pt-msmarco-10k-v1", "ptt5-base-pt-msmarco-100k-v1", "ptt5-base-en-pt-msmarco-10k-v1", "mt5-3B-mmarco-en-pt", "mt5-13b-mmarco-100k", "monoptt5-small", "monoptt5-base", "monoptt5-large", "monoptt5-3b"],
    # 'rankt5': ['rankt5-base', 'rankt5-large', 'rankt5-3b'],
    # 'listt5': ['listt5-base', 'listt5-3b'],
    # 'inranker': ['inranker-small', 'inranker-base', 'inranker-3b'],
    # 'transformer_ranker': ["mxbai-rerank-xsmall", "mxbai-rerank-base", "mxbai-rerank-large", "bge-reranker-base", "bge-reranker-large", "bge-reranker-v2-m3", "bce-reranker-base","jina-reranker-tiny",  "jina-reranker-turbo", "jina-reranker-base-multilingual","gte-multilingual-reranker-base", ],
    # 'transformer_ranker' : ["cross-encoder-mmarco-mMiniLMv2-L12-H384-v1", "nli-deberta-v3-large", "ms-marco-MiniLM-L-12-v2", "ms-marco-MiniLM-L-6-v2","ms-marco-MiniLM-L-4-v2", "ms-marco-MiniLM-L-2-v2", "ms-marco-TinyBERT-L-2-v2","ms-marco-electra-base", "ms-marco-TinyBERT-L-6", "ms-marco-TinyBERT-L-4","ms-marco-TinyBERT-L-2"],
    # 'llm_layerwise_ranker': ["meta-llama/Llama-3.2-1B","bge-multilingual-gemma2", "bge-reranker-v2-gemma", "bge-reranker-v2-minicpm-layerwise", "bge-reranker-v2.5-gemma2-lightweight"],
    # 'first_ranker':["First-Model"],
    # 'lit5dist': ["LiT5-Distill-base", "LiT5-Distill-large", "LiT5-Distill-xl", "LiT5-Distill-base-v2", "LiT5-Distill-large-v2", "LiT5-Distill-xl-v2"],
    # 'lit5score':   ["LiT5-Score-base", "LiT5-Score-large", "LiT5-Score-xl"],  
    # 'vicuna_reranker' : ["rank_vicuna_7b_v1", "rank_vicuna_7b_v1_noda", "rank_vicuna_7b_v1_fp16", "rank_vicuna_7b_v1_noda_fp16" ],      
    # 'zephyr_reranker': ["rank_zephyr_7b_v1_full"],
    # 'blender_reranker' : ["PairRM"],
    # 'splade_reranker' : ["splade-cocondenser"],
    # 'sentence_transformer_reranker':  ["all-MiniLM-L6-v2", "gtr-t5-base", "gtr-t5-large" , "gtr-t5-xl", "gtr-t5-xxl", "sentence-t5-base" , "sentence-t5-xl","sentence-t5-xxl","sentence-t5-large", "distilbert-multilingual-nli-stsb-quora-ranking", "msmarco-bert-co-condensor", "msmarco-roberta-base-v2" ], 
    # 'colbert_ranker': ['Colbert', 'jina-colbert-v1-en', 'mxbai-colbert-large-v1'],
    # 'monobert': ["monobert-large"],
    # 'llm2vec': ["Meta-Llama-31-8B", "Meta-Llama-3-8B", "Mistral-7B"],
    # 'twolar': ['twolar-xl'], 
    # 'echorank' : ['flan-t5-large', 'flan-t5-xl'],
    # 'incontext_reranker': ['llamav3.1-8b', 'Mistral-7B-Instruct-v0.2'], 

}

# Define datasets
datasets = ['dl19',"beir-covid",'dl20','beir-arguana',"beir-dbpedia", "beir-fever","beir-fiqa","beir-news","beir-nfc","beir-quora","beir-robust04","beir-scidocs","beir-scifact","beir-signal","beir-touche"] #'beir-arguana',   "beir-fever", "beir-fiqa", "beir-quora",  "beir-scidocs"]

# Log file path
log_file = "test_beir_dl.json"
if os.path.exists(log_file):
    with open(log_file, "r") as f:
        try:
            all_results = json.load(f)
        except json.JSONDecodeError:
            all_results = []
else:
    all_results = []

existing_results = {(r["retriever"], r["dataset"], r["method"], r["model_name"]) for r in all_results}

# Loop through methods and their models
for method, model_names in methods.items():
    print(f"Processing method: {method}, models: {model_names}")

    for model_name in model_names:
        for dataset_name in datasets:
            key = ("bm25", dataset_name, method, model_name)  # Unique identifier

            # **Skip processing if the model-dataset-method is already in log**
            if key in existing_results:
                print(f"✅ Skipping {method}-{model_name} on {dataset_name} (Already processed)")
                continue

            #try:
            print(f"🚀 Running {method}-{model_name} on {dataset_name}...")

            # **Download dataset**
            dataset = Dataset('bm25', dataset_name, 100)
            data = dataset.download(force_download=False)
            #data = Dataset.load_dataset(dataset_name, 100)
            # **Initialize and run reranker**
            model = Reranking(method=method, model_name=model_name, device = "cuda")
            start_time = time.time()
            model.rank(data)
            end_time = time.time()
            execution_time = end_time - start_time

            # **Calculate metrics**
            metrics = Metrics(data)

            map_cuts = [1, 5, 10, 20, 50, 100]

            mrr_cuts = [1, 5, 10, 20, 50, 100]
            before_ranking_metrics = metrics.calculate_trec_metrics(ndcg_cuts=[1, 5, 10, 20, 50, 100], map_cuts= map_cuts, mrr_cuts = mrr_cuts, qrel=dataset_name.replace("beir-",""), use_reordered=False)
            after_ranking_metrics = metrics.calculate_trec_metrics(ndcg_cuts=[1, 5, 10, 20, 50, 100], map_cuts= map_cuts, mrr_cuts= mrr_cuts , qrel=dataset_name.replace("beir-",""), use_reordered=True)

            # **Store results**
            result = {
                "retriever": "bm25",
                "dataset": dataset_name,
                "method": method,
                "model_name": model_name,
                "number_documents": str(len(data)),
                "before_ranking_metrics": before_ranking_metrics,
                "after_ranking_metrics": after_ranking_metrics,
                "execution_time": execution_time
            }
            all_results.append(result)

            # **Save results to log file after each iteration**
            with open(log_file, "w") as f:
                json.dump(all_results, f, indent=4)

            # Print results for debugging
            print(f"✔ Completed {method}-{model_name} on {dataset_name}")
            print(f"Before Ranking Metrics: {before_ranking_metrics}")
            print(f"After Ranking Metrics: {after_ranking_metrics}")
            print("#" * 100)
            import torch
            import gc
            torch.cuda.empty_cache()
            gc.collect()



print("✅ All results saved to:", log_file)

[amitbcp]

Thanks @abdoelsayed2016 ! Let me try this!
Also when you get time please share how to pass custom Qrel for custom dataset.

For the other non-bier datasets, it might be worth considering creating and including Quells to enable MRR/nDCG metrics as to show improved retrieval/reranking on any dataset/benchmark, these datasets are used

[abdoelsayed2016]

Hello @amitbcp

I’ve added an example showing how to use a custom dataset and a custom qrel. You can find it here: https://github.com/DataScienceUIBK/Rankify/tree/main/examples/custom_datasets