Fixed FIGER dataset problem where there are multiple possible labels for an...

Fixed FIGER dataset problem where there are multiple possible labels for an entity and only one is counted as correct

src/experiments/GLiNER_evalutaion/evaluation.py

@@ -6,6 +6,6 @@ NEC evaluation for GLiNER is part of /NEC_evaluation.
 from src.metrics import NER_metrics, read_NER_metrics
 
 for dataset in ["CoNLL", "FIGER-coarse", "FIGER-fine"]:
-    NER_metrics("GLiNER", dataset, f"results_{dataset}", test_instances=1000)
+    #NER_metrics("GLiNER", dataset, f"results_{dataset}", test_instances=1000)
     print(f"\nResults {dataset}:")
     read_NER_metrics(f"results_{dataset}")

src/experiments/NEC_evaluation/context_sensitivity.py

-
-
 import data.data_manager as data_manager
 from src.common_interface import classify_entity
 
+
 def run_context_analysis(model_name, dataset, num_sentences):
     labels = data_manager.get_labels(dataset)
     data = data_manager.get_annotated_sentences(dataset, num_sentences)
@@ -30,4 +29,4 @@ def run_context_analysis(model_name, dataset, num_sentences):
             print(f"Predicted: {predicted}, True: {entity[1]}")
 
         
-run_context_analysis("T5-NLI", "FIGER-coarse", 50)
+run_context_analysis("Llama-3.1-8B", "FIGER-coarse", 50)

src/experiments/NEC_evaluation/evaluation.py

@@ -40,25 +40,36 @@ def run_NEC_tests(model_name, dataset, results_dir, test_instances=10):
 
         for idx, entry in enumerate(tqdm(data, desc="Processing Instances", unit="instance")):
             sentence = entry[0]
+            entity_dict = {}  # If there are multiple labels assigned to one entity,
+            # it should count as correct if the model predicts one of them.
             for entity in entry[1]:
-                predicted_label = classify_entity(model_name, sentence, entity[0], labels)
-                correct = predicted_label == entity[1]
-
-                csv_writer.writerow([idx, model_name, dataset, sentence, entity[0], entity[1], predicted_label, correct])
-
-                # Log stuff
-                txt_file.write(f"Instance {idx}:\n")
-                txt_file.write(f"Sentence: {sentence}\n")
-                txt_file.write(f"Entity: {entity[0]}\n")
-                txt_file.write(f"True Label: {entity[1]}\n")
-                txt_file.write(f"Predicted Label: {predicted_label}\n")
+                entity_name = entity[0]
+                true_label = entity[1]
+
+                if entity_name not in entity_dict:
+                    entity_dict[entity_name] = set()
+                entity_dict[entity_name].add(true_label)
+
+            for entity_name, true_labels in entity_dict.items():
+                predicted_label = classify_entity(model_name, sentence, entity_name, labels)
+                correct = predicted_label in true_labels
+
+                csv_writer.writerow(
+                    [idx, model_name, dataset, sentence, entity_name, ", ".join(true_labels), predicted_label, correct])
+
+                # Log results
+                txt_file.write(f"\nInstance {idx}:")
+                txt_file.write(f"\nSentence: {sentence}")
+                txt_file.write(f"\nEntity: {entity_name}")
+                txt_file.write(f"\nTrue Label(s): {', '.join(true_labels)}")
+                txt_file.write(f"\nPredicted Label: {predicted_label}")
                 txt_file.write("-" * 50 + "\n\n")
 
     print(f"Results saved to:\n  CSV: {csv_filename}\n  TXT: {txt_filename}")
 
 
 def run_NEC_tests_all():
-    models = ["DeepSeek-R1-Distill-Qwen-32B"]  # "GLiNER", "Llama-3.1-8B", "T5-NLI", "T5-MLM-label", "T5-MLM-entity", "Word2Vec"
+    models = ["GLiNER", "Llama-3.1-8B", "T5-NLI", "T5-MLM-label", "T5-MLM-entity", "DeepSeek-R1-Distill-Qwen-32B"]  # "GLiNER", "Llama-3.1-8B", "T5-NLI", "T5-MLM-label", "T5-MLM-entity", "Word2Vec"
     datasets = ["CoNLL", "FIGER-coarse", "FIGER-fine"]  # "Pile-NER-type"]
     for model in models:
         for dataset in datasets: