Replace eval metric with lenskit TopN

pyproject.toml

@@ -11,6 +11,7 @@ requires-python = ">=3.10"
 keywords = []
 authors = [{ name = "Karl Higley", email = "khigley@umn.edu" }]
 dependencies = [
+  "lenskit==0.14.*",
   "nltk>=3.8,<4",
   "numpy>=1.26,<2",
   "pandas==2.*",

src/poprox_recommender/paths.py

@@ -2,4 +2,4 @@
 
 
 def project_root() -> Path:
-    return Path(__file__).parent.parent
+    return Path(__file__).parent.parent.parent

src/poprox_recommender/test_json.py

@@ -2,14 +2,15 @@
 import json
 import os
 import sys
-sys.path.append('../')
+from lenskit.metrics import topn
+sys.path.append('src')
 import torch as th
 from safetensors.torch import load_file
 from uuid import UUID
 import numpy as np
 import pandas as pd
 from tqdm import tqdm
-from poprox_recommender.domain import Article, ClickHistory
+from poprox_concepts import Article, ClickHistory
 from poprox_recommender.default import select_articles
 from poprox_recommender.paths import project_root
 
@@ -20,7 +21,7 @@ def load_model(device_name=None):
     if device_name is None:
         device_name = "cuda" if th.cuda.is_available() else "cpu"
 
-    load_path = f"/home/XLL1713/POPROX/engine0/NRMS_bert/NRMS_BERT_checkpoint/model.safetensors"
+    load_path = f"{project_root()}/models/model.safetensors"
     checkpoint = load_file(load_path)
 
     return checkpoint, device_name
@@ -75,7 +76,7 @@ def compute_ndcg(y_pred, y_true, k):
 def recsys_metric(recommendations, row_index, news_struuid_ID ):
     # recommendations {account id (uuid): LIST[Article]}
     # use the url of Article
-    impressions_truth = pd.read_table("/home/XLL1713/POPROX/engine0/NRMS_bert/data/mind/large/val/behaviors.tsv",
+    impressions_truth = pd.read_table(f"{project_root()}/data/test_mind_large/behaviors.tsv",
                                     header='infer',
                                     usecols=range(5),
                                     names=[
@@ -86,12 +87,29 @@ def recsys_metric(recommendations, row_index, news_struuid_ID ):
     account_id = list(recommendations.keys())[0]
     recommended_list = recommendations[account_id]
     recommended_list = [news_struuid_ID[item.url] for item in recommended_list]
-
-    single_mrr = compute_mrr(recommended_list, impressions_truth)
-    single_ndcg5 = compute_ndcg(recommended_list, impressions_truth, 5)
-    single_ndcg10 = compute_ndcg(recommended_list, impressions_truth, 10)
-
-    return single_ndcg5, single_ndcg10, single_mrr
+
+
+    recs = pd.DataFrame({
+        'item': recommended_list
+    })
+
+    truth = pd.DataFrame.from_records(
+        (
+            (row.split('-')[0], int(row.split('-')[1]))
+            for row in impressions_truth
+        ),
+        columns=['item', 'rating']
+    ).set_index('item')
+
+    single_rr = topn.recip_rank(recs, truth)
+    single_ndcg5 = topn.ndcg(recs, truth, k=5)
+    single_ndcg10 = topn.ndcg(recs, truth, k=10)
+
+    # single_rr = compute_mrr(recommended_list, impressions_truth)
+    # single_ndcg5 = compute_ndcg(recommended_list, impressions_truth, 5)
+    # single_ndcg10 = compute_ndcg(recommended_list, impressions_truth, 10)
+
+    return single_ndcg5, single_ndcg10, single_rr
 
 if __name__ == '__main__':
     '''
@@ -100,18 +118,18 @@ def recsys_metric(recommendations, row_index, news_struuid_ID ):
     MODEL, DEVICE = load_model()
     TOKEN_MAPPING = 'distilbert-base-uncased'  # can be modified
 
-    with open('/home/XLL1713/POPROX/poprox-recommender/tests/news_uuid_ID.json', 'r') as json_file:
+    with open(f"{project_root()}/data/val_mind_large/news_uuid_ID.json", 'r') as json_file:
         news_struuid_ID = json.load(json_file)
 
     # load the mind test json file
-    with open('/home/XLL1713/POPROX/poprox-recommender/tests/mind_test.json', 'r') as json_file:
+    with open(f"{project_root()}/data/val_mind_large/mind_test.json", 'r') as json_file:
         mind_data = json.load(json_file)
 
     ndcg5 = []
     ndcg10 = []
     mrr = []
 
-    for impression_idx in range(len(mind_data)): # one by one
+    for impression_idx in range(10): # one by one
 
         request_body = mind_data[impression_idx]
 
@@ -135,15 +153,15 @@ def recsys_metric(recommendations, row_index, news_struuid_ID ):
 
         single_ndcg5, single_ndcg10, single_mrr = recsys_metric(recommendations, impression_idx, news_struuid_ID )
         # recommendations {account id (uuid): LIST[Article]}
-        print(f"----------------evaluation using the first {impression_idx + 1} is ndcg5 = {single_ndcg5}, ndcg10 = {single_ndcg10}, mrr = {single_mrr}")
+        print(f"----------------evaluation using the first {impression_idx + 1} is NDCG@5 = {single_ndcg5}, NDCG@10 = {single_ndcg10}, RR = {single_mrr}")
 
         ndcg5.append(single_ndcg5)
         ndcg10.append(single_ndcg10)
         mrr.append(single_mrr)
 
 
 
-    print(f"Offline evaluation metrics on MIND data: ndcg5 = {np.mean(ndcg5)}, ndcg10 = {np.mean(ndcg10)}, mrr = {np.mean(mrr)}")
+    print(f"Offline evaluation metrics on MIND data: NDCG@5 = {np.mean(ndcg5)}, NDCG@10 = {np.mean(ndcg10)}, MRR = {np.mean(mrr)}")
 
 
         #response = {"statusCode": 200, "body": json.dump(body, default=custom_encoder)}