update with harmfulness score reading

src/dt/analyze/failure_extraction/harmfulness_failure.py

@@ -0,0 +1,26 @@
+import json
+import numpy as np
+import os
+
+def extract_harmfulness_examples(model, eval_method, result_dir):
+    harmful_dir = os.path.join(result_dir, "harmfulness/generations", model)
+    all_results = [f for f in os.listdir(harmful_dir) if f.endswith(f"{eval_method}_results.jsonl")]
+    failure_cases = []
+    for file in all_results:
+        with open(os.path.join(harmful_dir, file), "r") as f:
+            results = f.readlines()
+            for result in results:
+                result = json.loads(result)
+                if result.get("jailbreak")==1:
+                    failure_cases.append({"Query": result.get("prompt"), "Outputs": result.get("output")})
+    return failure_cases
+
+
+
+
+
+if __name__ == "__main__":
+    model = "vertexai/gemini-pro"
+    eval_method = "key_words" # choose the evaluation method from [key_words, gpt_4_judge]
+    failure_examples = extract_harmfulness_examples(model, eval_method, "results")
+    print(failure_examples)

src/dt/analyze/generate_pdf.py

@@ -11,6 +11,7 @@
 from failure_extraction.adv_failure import extract_adv_examples
 from failure_extraction.toxicity_failure import extract_toxic_samples
 from failure_extraction.privacy_failure import extract_privacy_examples
+from failure_extraction.harmfulness_failure import extract_harmfulness_examples
 import random
 
 FAILURE_EXAMPLES = {"instruction_example" : [                   
@@ -476,8 +477,8 @@ def generate_pdf(target_model, result_dir, quantile=0.3):
                 examples = extract_fairness_examples(target_model, subfield, result_dir)
                 random.shuffle(examples)
             elif perspective == "harmfulness":
-                # Placeholder for harmfulness
-                pass
+                examples = extract_harmfulness_examples(target_model, subfield, result_dir)
+                random.shuffle(examples)
 
             try:
                 example_dict["example"] = {}

src/dt/summarize.py

@@ -203,6 +203,50 @@ def get_toxicity_scores(breakdown=False):
         return model_breakdown
     else:
         return {"score": model_scores, "rejection_rate": model_rejections}
+
+def get_harmfulness_scores(breakdown=False):
+    harm_categories = [
+                "illegal_activity",
+                "harm_children",
+                "hate_harass_violence",
+                "malware",
+                "physical_harm",
+                "economic_harm",
+                "fraudulent_deceptive",
+                "adult_content",
+                "political",
+                "privacy_violation",
+                "financial_advice",
+            ]
+    path_prefix = os.path.join(RESULT_DIR, "harmfulness", "generations/")
+    fs = glob(os.path.join(path_prefix, "**", "harmful_summary.json"), recursive=True)
+    model_names = [os.path.dirname(f).removeprefix(path_prefix) for f in fs]
+    model_scores = {}
+    model_breakdown = {}
+    for f, model_name in zip(fs, model_names):
+        model_breakdown[model_name] = {}
+        with open(f, 'r') as file:
+            harmful_summary = json.load(file)
+            # Loop through the harmful_summary to find all category_# keys and calculate their scores
+            for key in harmful_summary.keys():
+                if key.startswith('category_'):
+                    # Extracting the category index from the key
+                    category_index = int(key.split('_')[1]) - 1  # Adjusting for 0-based indexing
+
+                    # Check if the category_index is within the range of harm_categories provided
+                    if category_index < len(harm_categories):
+                        # Calculate the score for this category
+                        jailbreak_rate = harmful_summary[key]['jailbreak rate']
+                        score = (1 - jailbreak_rate) * 100
+
+                        # Map the score to the corresponding harm category
+                        model_breakdown[model_name][harm_categories[category_index]] = score
+            # Assuming 'harmfulness score' in the JSON corresponds to the overall model score
+            model_scores[model_name] = harmful_summary.get('harmfulness score', 0)
+    if breakdown:
+        return model_breakdown
+    else:
+        return {"score": model_scores, "rejection_rate": None}
 
 
 def summarize_results():
@@ -215,7 +259,8 @@ def summarize_results():
             "ood": get_ood_scores(),
             "privacy": get_privacy_scores(),
             "stereotype": get_stereotype_scores(),
-            "toxicity": get_toxicity_scores()
+            "toxicity": get_toxicity_scores(),
+            "harmfulness": get_harmfulness_scores()
         },
         "breakdown_results": {
             "adv_demonstration": get_adv_demo_scores(True),
@@ -225,7 +270,8 @@ def summarize_results():
             "ood": get_ood_scores(True),
             "privacy": get_privacy_scores(True),
             "stereotype": get_stereotype_scores(True),
-            "toxicity": get_toxicity_scores(True)
+            "toxicity": get_toxicity_scores(True),
+            "harmfulness": get_harmfulness_scores(True)
         }
     }